diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
36 files changed, 1155 insertions, 1557 deletions
diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index 4892ace53e3..28765af20ad 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -4,7 +4,7 @@ use rustc_hir::lang_items::LangItem; use rustc_index::IndexVec; use rustc_middle::mir::*; use rustc_middle::mir::{ - interpret::{ConstValue, Scalar}, + interpret::Scalar, visit::{PlaceContext, Visitor}, }; use rustc_middle::ty::{Ty, TyCtxt, TypeAndMut}; @@ -181,13 +181,10 @@ fn insert_alignment_check<'tcx>( // Subtract 1 from the alignment to get the alignment mask let alignment_mask = local_decls.push(LocalDecl::with_source_info(tcx.types.usize, source_info)).into(); - let one = Operand::Constant(Box::new(Constant { + let one = Operand::Constant(Box::new(ConstOperand { span: source_info.span, user_ty: None, - literal: ConstantKind::Val( - ConstValue::Scalar(Scalar::from_target_usize(1, &tcx)), - tcx.types.usize, - ), + const_: Const::Val(ConstValue::Scalar(Scalar::from_target_usize(1, &tcx)), tcx.types.usize), })); block_data.statements.push(Statement { source_info, @@ -213,13 +210,10 @@ fn insert_alignment_check<'tcx>( // Check if the alignment bits are all zero let is_ok = local_decls.push(LocalDecl::with_source_info(tcx.types.bool, source_info)).into(); - let zero = Operand::Constant(Box::new(Constant { + let zero = Operand::Constant(Box::new(ConstOperand { span: source_info.span, user_ty: None, - literal: ConstantKind::Val( - ConstValue::Scalar(Scalar::from_target_usize(0, &tcx)), - tcx.types.usize, - ), + const_: Const::Val(ConstValue::Scalar(Scalar::from_target_usize(0, &tcx)), tcx.types.usize), })); block_data.statements.push(Statement { source_info, diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index d5af321d726..bacabc62ee4 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -142,9 +142,9 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { fn visit_operand(&mut self, op: &Operand<'tcx>, location: Location) { if let Operand::Constant(constant) = op { - let maybe_uneval = match constant.literal { - ConstantKind::Val(..) | ConstantKind::Ty(_) => None, - ConstantKind::Unevaluated(uv, _) => Some(uv), + let maybe_uneval = match constant.const_ { + Const::Val(..) | Const::Ty(_) => None, + Const::Unevaluated(uv, _) => Some(uv), }; if let Some(uv) = maybe_uneval { diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index f662ce645b0..40cd2825408 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -4,7 +4,7 @@ use rustc_middle::{ mir::{ visit::{PlaceContext, Visitor}, - Body, Constant, Local, Location, Operand, Rvalue, StatementKind, VarDebugInfoContents, + Body, ConstOperand, Local, Location, Operand, Rvalue, StatementKind, VarDebugInfoContents, }, ty::TyCtxt, }; @@ -45,7 +45,7 @@ struct LocalUseVisitor { local_assignment_locations: IndexVec<Local, Option<Location>>, } -fn find_optimization_opportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Constant<'tcx>)> { +fn find_optimization_opportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, ConstOperand<'tcx>)> { let mut visitor = LocalUseVisitor { local_mutating_uses: IndexVec::from_elem(0, &body.local_decls), local_assignment_locations: IndexVec::from_elem(None, &body.local_decls), diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index e175f22d7a9..fd2d37dbea5 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -96,10 +96,10 @@ impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> { let (discr, targets) = target_bb_terminator.kind.as_switch()?; if discr.place() == Some(*place) { let switch_ty = place.ty(self.body.local_decls(), self.tcx).ty; + debug_assert_eq!(switch_ty, _const.ty()); // We now know that the Switch matches on the const place, and it is statementless // Now find which value in the Switch matches the const value. - let const_value = - _const.literal.try_eval_bits(self.tcx, self.param_env, switch_ty)?; + let const_value = _const.const_.try_eval_bits(self.tcx, self.param_env)?; let target_to_use_in_goto = targets.target_for_value(const_value); self.optimizations.push(OptimizationToApply { bb_with_goto: location.block, diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 5f135e96980..4fc78b28580 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -15,14 +15,15 @@ use rustc_middle::mir::visit::{ use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::{self, GenericArgs, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt}; -use rustc_span::{def_id::DefId, Span, DUMMY_SP}; +use rustc_span::{def_id::DefId, Span}; use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout}; use rustc_target::spec::abi::Abi as CallAbi; +use crate::dataflow_const_prop::Patch; use crate::MirPass; use rustc_const_eval::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, ConstValue, FnArg, Frame, ImmTy, - Immediate, InterpCx, InterpResult, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, StackPopCleanup, + self, compile_time_machine, AllocId, ConstAllocation, FnArg, Frame, ImmTy, Immediate, InterpCx, + InterpResult, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, StackPopCleanup, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -32,32 +33,30 @@ const MAX_ALLOC_LIMIT: u64 = 1024; /// Macro for machine-specific `InterpError` without allocation. /// (These will never be shown to the user, but they help diagnose ICEs.) -macro_rules! throw_machine_stop_str { - ($($tt:tt)*) => {{ - // We make a new local type for it. The type itself does not carry any information, - // but its vtable (for the `MachineStopType` trait) does. - #[derive(Debug)] - struct Zst; - // Printing this type shows the desired string. - impl std::fmt::Display for Zst { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, $($tt)*) - } +pub(crate) macro throw_machine_stop_str($($tt:tt)*) {{ + // We make a new local type for it. The type itself does not carry any information, + // but its vtable (for the `MachineStopType` trait) does. + #[derive(Debug)] + struct Zst; + // Printing this type shows the desired string. + impl std::fmt::Display for Zst { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, $($tt)*) } + } - impl rustc_middle::mir::interpret::MachineStopType for Zst { - fn diagnostic_message(&self) -> rustc_errors::DiagnosticMessage { - self.to_string().into() - } - - fn add_args( - self: Box<Self>, - _: &mut dyn FnMut(std::borrow::Cow<'static, str>, rustc_errors::DiagnosticArgValue<'static>), - ) {} + impl rustc_middle::mir::interpret::MachineStopType for Zst { + fn diagnostic_message(&self) -> rustc_errors::DiagnosticMessage { + self.to_string().into() } - throw_machine_stop!(Zst) - }}; -} + + fn add_args( + self: Box<Self>, + _: &mut dyn FnMut(std::borrow::Cow<'static, str>, rustc_errors::DiagnosticArgValue<'static>), + ) {} + } + throw_machine_stop!(Zst) +}} pub struct ConstProp; @@ -85,9 +84,9 @@ impl<'tcx> MirPass<'tcx> for ConstProp { return; } - let is_generator = tcx.type_of(def_id.to_def_id()).instantiate_identity().is_generator(); // FIXME(welseywiser) const prop doesn't work on generators because of query cycles // computing their layout. + let is_generator = def_kind == DefKind::Generator; if is_generator { trace!("ConstProp skipped for generator {:?}", def_id); return; @@ -95,33 +94,22 @@ impl<'tcx> MirPass<'tcx> for ConstProp { trace!("ConstProp starting for {:?}", def_id); - let dummy_body = &Body::new( - body.source, - (*body.basic_blocks).to_owned(), - body.source_scopes.clone(), - body.local_decls.clone(), - Default::default(), - body.arg_count, - Default::default(), - body.span, - body.generator_kind(), - body.tainted_by_errors, - ); - // FIXME(oli-obk, eddyb) Optimize locals (or even local paths) to hold // constants, instead of just checking for const-folding succeeding. // That would require a uniform one-def no-mutation analysis // and RPO (or recursing when needing the value of a local). - let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); + let mut optimization_finder = ConstPropagator::new(body, tcx); // Traverse the body in reverse post-order, to ensure that `FullConstProp` locals are // assigned before being read. - let rpo = body.basic_blocks.reverse_postorder().to_vec(); - for bb in rpo { - let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; + for &bb in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[bb]; optimization_finder.visit_basic_block_data(bb, data); } + let mut patch = optimization_finder.patch; + patch.visit_body_preserves_cfg(body); + trace!("ConstProp done for {:?}", def_id); } } @@ -145,14 +133,17 @@ impl ConstPropMachine<'_, '_> { impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { compile_time_machine!(<'mir, 'tcx>); + const PANIC_ON_ALLOC_FAIL: bool = true; // all allocations are small (see `MAX_ALLOC_LIMIT`) + const POST_MONO_CHECKS: bool = false; // this MIR is still generic! + type MemoryKind = !; #[inline(always)] fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> CheckAlignment { // We do not check for alignment to avoid having to carry an `Align` - // in `ConstValue::ByRef`. + // in `ConstValue::Indirect`. CheckAlignment::No } @@ -219,7 +210,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _bin_op: BinOp, _left: &ImmTy<'tcx>, _right: &ImmTy<'tcx>, - ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> { + ) -> InterpResult<'tcx, (ImmTy<'tcx>, bool)> { // We can't do this because aliasing of memory can differ between const eval and llvm throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") } @@ -301,6 +292,7 @@ struct ConstPropagator<'mir, 'tcx> { tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, local_decls: &'mir IndexSlice<Local, LocalDecl<'tcx>>, + patch: Patch<'tcx>, } impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { @@ -334,11 +326,7 @@ impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { } impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { - fn new( - body: &Body<'tcx>, - dummy_body: &'mir Body<'tcx>, - tcx: TyCtxt<'tcx>, - ) -> ConstPropagator<'mir, 'tcx> { + fn new(body: &'mir Body<'tcx>, tcx: TyCtxt<'tcx>) -> ConstPropagator<'mir, 'tcx> { let def_id = body.source.def_id(); let args = &GenericArgs::identity_for_item(tcx, def_id); let param_env = tcx.param_env_reveal_all_normalized(def_id); @@ -369,7 +357,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ecx.push_stack_frame( Instance::new(def_id, args), - dummy_body, + body, &ret, StackPopCleanup::Root { cleanup: false }, ) @@ -384,7 +372,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ecx.frame_mut().locals[local].make_live_uninit(); } - ConstPropagator { ecx, tcx, param_env, local_decls: &dummy_body.local_decls } + let patch = Patch::new(tcx); + ConstPropagator { ecx, tcx, param_env, local_decls: &body.local_decls, patch } } fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { @@ -421,12 +410,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ecx.machine.written_only_inside_own_block_locals.remove(&local); } - fn propagate_operand(&mut self, operand: &mut Operand<'tcx>) { - if let Some(place) = operand.place() && let Some(op) = self.replace_with_const(place) { - *operand = op; - } - } - fn check_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Option<()> { // Perform any special handling for specific Rvalue types. // Generally, checks here fall into one of two categories: @@ -542,16 +525,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - /// Creates a new `Operand::Constant` from a `Scalar` value - fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>) -> Operand<'tcx> { - Operand::Constant(Box::new(Constant { - span: DUMMY_SP, - user_ty: None, - literal: ConstantKind::from_scalar(self.tcx, scalar, ty), - })) - } - - fn replace_with_const(&mut self, place: Place<'tcx>) -> Option<Operand<'tcx>> { + fn replace_with_const(&mut self, place: Place<'tcx>) -> Option<Const<'tcx>> { // This will return None if the above `const_prop` invocation only "wrote" a // type whose creation requires no write. E.g. a generator whose initial state // consists solely of uninitialized memory (so it doesn't capture any locals). @@ -561,31 +535,26 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } trace!("replacing {:?} with {:?}", place, value); - // FIXME> figure out what to do when read_immediate_raw fails + // FIXME: figure out what to do when read_immediate_raw fails let imm = self.ecx.read_immediate_raw(&value).ok()?; let Right(imm) = imm else { return None }; match *imm { Immediate::Scalar(scalar) if scalar.try_to_int().is_ok() => { - Some(self.operand_from_scalar(scalar, value.layout.ty)) + Some(Const::from_scalar(self.tcx, scalar, value.layout.ty)) } Immediate::ScalarPair(l, r) if l.try_to_int().is_ok() && r.try_to_int().is_ok() => { - let alloc = self + let alloc_id = self .ecx .intern_with_temp_alloc(value.layout, |ecx, dest| { ecx.write_immediate(*imm, dest) }) .ok()?; - let literal = ConstantKind::Val( - ConstValue::ByRef { alloc, offset: Size::ZERO }, + Some(Const::Val( + ConstValue::Indirect { alloc_id, offset: Size::ZERO }, value.layout.ty, - ); - Some(Operand::Constant(Box::new(Constant { - span: DUMMY_SP, - user_ty: None, - literal, - }))) + )) } // Scalars or scalar pairs that contain undef values are assumed to not have // successfully evaluated and are thus not propagated. @@ -727,40 +696,29 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { } } -impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx - } - - fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { +impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { + fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { self.super_operand(operand, location); - self.propagate_operand(operand) + if let Some(place) = operand.place() && let Some(value) = self.replace_with_const(place) { + self.patch.before_effect.insert((location, place), value); + } } - fn process_projection_elem( + fn visit_projection_elem( &mut self, + _: PlaceRef<'tcx>, elem: PlaceElem<'tcx>, - _: Location, - ) -> Option<PlaceElem<'tcx>> { + _: PlaceContext, + location: Location, + ) { if let PlaceElem::Index(local) = elem - && let Some(value) = self.get_const(local.into()) - && let Some(imm) = value.as_mplace_or_imm().right() - && let Immediate::Scalar(scalar) = *imm - && let Ok(offset) = scalar.to_target_usize(&self.tcx) - && let Some(min_length) = offset.checked_add(1) + && let Some(value) = self.replace_with_const(local.into()) { - Some(PlaceElem::ConstantIndex { offset, min_length, from_end: false }) - } else { - None + self.patch.before_effect.insert((location, local.into()), value); } } - fn visit_assign( - &mut self, - place: &mut Place<'tcx>, - rvalue: &mut Rvalue<'tcx>, - location: Location, - ) { + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { self.super_assign(place, rvalue, location); let Some(()) = self.check_rvalue(rvalue) else { return }; @@ -773,11 +731,11 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { if let Some(()) = self.eval_rvalue_with_identities(rvalue, *place) { // If this was already an evaluated constant, keep it. if let Rvalue::Use(Operand::Constant(c)) = rvalue - && let ConstantKind::Val(..) = c.literal + && let Const::Val(..) = c.const_ { trace!("skipping replace of Rvalue::Use({:?} because it is already a const", c); } else if let Some(operand) = self.replace_with_const(*place) { - *rvalue = Rvalue::Use(operand); + self.patch.assignments.insert(location, operand); } } else { // Const prop failed, so erase the destination, ensuring that whatever happens @@ -801,7 +759,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } - fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { trace!("visit_statement: {:?}", statement); // We want to evaluate operands before any change to the assigned-to value, @@ -845,7 +803,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } - fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { + fn visit_basic_block_data(&mut self, block: BasicBlock, data: &BasicBlockData<'tcx>) { self.super_basic_block_data(block, data); // We remove all Locals which are restricted in propagation to their containing blocks and diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index b52827a1e88..64e262c6c93 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -105,25 +105,12 @@ impl<'tcx> MirLint<'tcx> for ConstProp { trace!("ConstProp starting for {:?}", def_id); - let dummy_body = &Body::new( - body.source, - (*body.basic_blocks).to_owned(), - body.source_scopes.clone(), - body.local_decls.clone(), - Default::default(), - body.arg_count, - Default::default(), - body.span, - body.generator_kind(), - body.tainted_by_errors, - ); - // FIXME(oli-obk, eddyb) Optimize locals (or even local paths) to hold // constants, instead of just checking for const-folding succeeding. // That would require a uniform one-def no-mutation analysis // and RPO (or recursing when needing the value of a local). - let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); - optimization_finder.visit_body(body); + let mut linter = ConstPropagator::new(body, tcx); + linter.visit_body(body); trace!("ConstProp done for {:?}", def_id); } @@ -169,11 +156,7 @@ impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { } impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { - fn new( - body: &Body<'tcx>, - dummy_body: &'mir Body<'tcx>, - tcx: TyCtxt<'tcx>, - ) -> ConstPropagator<'mir, 'tcx> { + fn new(body: &'mir Body<'tcx>, tcx: TyCtxt<'tcx>) -> ConstPropagator<'mir, 'tcx> { let def_id = body.source.def_id(); let args = &GenericArgs::identity_for_item(tcx, def_id); let param_env = tcx.param_env_reveal_all_normalized(def_id); @@ -204,7 +187,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ecx.push_stack_frame( Instance::new(def_id, args), - dummy_body, + body, &ret, StackPopCleanup::Root { cleanup: false }, ) @@ -298,7 +281,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } /// Returns the value, if any, of evaluating `c`. - fn eval_constant(&mut self, c: &Constant<'tcx>, location: Location) -> Option<OpTy<'tcx>> { + fn eval_constant(&mut self, c: &ConstOperand<'tcx>, location: Location) -> Option<OpTy<'tcx>> { // FIXME we need to revisit this for #67176 if c.has_param() { return None; @@ -310,7 +293,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // that the `RevealAll` pass has happened and that the body's consts // are normalized, so any call to resolve before that needs to be // manually normalized. - let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.literal).ok()?; + let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.const_).ok()?; self.use_ecx(location, |this| this.ecx.eval_mir_constant(&val, Some(c.span), None)) } @@ -339,7 +322,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn check_unary_op(&mut self, op: UnOp, arg: &Operand<'tcx>, location: Location) -> Option<()> { if let (val, true) = self.use_ecx(location, |this| { let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?; - let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, &val)?; + let (_res, overflow) = this.ecx.overflowing_unary_op(op, &val)?; Ok((val, overflow)) })? { // `AssertKind` only has an `OverflowNeg` variant, so make sure that is @@ -407,7 +390,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if let (Some(l), Some(r)) = (l, r) { // The remaining operators are handled through `overflowing_binary_op`. if self.use_ecx(location, |this| { - let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, &l, &r)?; + let (_res, overflow) = this.ecx.overflowing_binary_op(op, &l, &r)?; Ok(overflow) })? { let source_info = self.body().source_info(location); @@ -597,7 +580,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { self.super_operand(operand, location); } - fn visit_constant(&mut self, constant: &Constant<'tcx>, location: Location) { + fn visit_constant(&mut self, constant: &ConstOperand<'tcx>, location: Location) { trace!("visit_constant: {:?}", constant); self.super_constant(constant, location); self.eval_constant(constant, location); diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 3d442e5dca9..d56d4ad4f1e 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -1,10 +1,8 @@ use super::Error; -use super::debug; use super::graph; use super::spans; -use debug::{DebugCounters, NESTED_INDENT}; use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops}; use spans::CoverageSpan; @@ -16,6 +14,8 @@ use rustc_middle::mir::coverage::*; use std::fmt::{self, Debug}; +const NESTED_INDENT: &str = " "; + /// The coverage counter or counter expression associated with a particular /// BCB node or BCB edge. #[derive(Clone)] @@ -75,8 +75,6 @@ pub(super) struct CoverageCounters { /// BCB/edge, but are needed as operands to more complex expressions. /// These are always [`BcbCounter::Expression`]. pub(super) intermediate_expressions: Vec<BcbCounter>, - - pub debug_counters: DebugCounters, } impl CoverageCounters { @@ -91,17 +89,9 @@ impl CoverageCounters { bcb_edge_counters: FxHashMap::default(), bcb_has_incoming_edge_counters: BitSet::new_empty(num_bcbs), intermediate_expressions: Vec::new(), - - debug_counters: DebugCounters::new(), } } - /// Activate the `DebugCounters` data structures, to provide additional debug formatting - /// features when formatting [`BcbCounter`] (counter) values. - pub fn enable_debug(&mut self) { - self.debug_counters.enable(); - } - /// Makes [`BcbCounter`] `Counter`s and `Expressions` for the `BasicCoverageBlock`s directly or /// indirectly associated with `CoverageSpans`, and accumulates additional `Expression`s /// representing intermediate values. @@ -113,44 +103,18 @@ impl CoverageCounters { MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(coverage_spans) } - fn make_counter<F>(&mut self, debug_block_label_fn: F) -> BcbCounter - where - F: Fn() -> Option<String>, - { - let counter = BcbCounter::Counter { id: self.next_counter() }; - if self.debug_counters.is_enabled() { - self.debug_counters.add_counter(&counter, (debug_block_label_fn)()); - } - counter + fn make_counter(&mut self) -> BcbCounter { + let id = self.next_counter(); + BcbCounter::Counter { id } } - fn make_expression<F>( - &mut self, - lhs: Operand, - op: Op, - rhs: Operand, - debug_block_label_fn: F, - ) -> BcbCounter - where - F: Fn() -> Option<String>, - { + fn make_expression(&mut self, lhs: Operand, op: Op, rhs: Operand) -> BcbCounter { let id = self.next_expression(); - let expression = BcbCounter::Expression { id, lhs, op, rhs }; - if self.debug_counters.is_enabled() { - self.debug_counters.add_counter(&expression, (debug_block_label_fn)()); - } - expression + BcbCounter::Expression { id, lhs, op, rhs } } pub fn make_identity_counter(&mut self, counter_operand: Operand) -> BcbCounter { - let some_debug_block_label = if self.debug_counters.is_enabled() { - self.debug_counters.some_block_label(counter_operand).cloned() - } else { - None - }; - self.make_expression(counter_operand, Op::Add, Operand::Zero, || { - some_debug_block_label.clone() - }) + self.make_expression(counter_operand, Op::Add, Operand::Zero) } /// Counter IDs start from one and go up. @@ -367,12 +331,8 @@ impl<'a> MakeBcbCounters<'a> { branch_counter_operand, Op::Add, sumup_counter_operand, - || None, - ); - debug!( - " [new intermediate expression: {}]", - self.format_counter(&intermediate_expression) ); + debug!(" [new intermediate expression: {:?}]", intermediate_expression); let intermediate_expression_operand = intermediate_expression.as_operand(); self.coverage_counters.intermediate_expressions.push(intermediate_expression); some_sumup_counter_operand.replace(intermediate_expression_operand); @@ -394,9 +354,8 @@ impl<'a> MakeBcbCounters<'a> { branching_counter_operand, Op::Subtract, sumup_counter_operand, - || Some(format!("{expression_branch:?}")), ); - debug!("{:?} gets an expression: {}", expression_branch, self.format_counter(&expression)); + debug!("{:?} gets an expression: {:?}", expression_branch, expression); let bcb = expression_branch.target_bcb; if expression_branch.is_only_path_to_target() { self.coverage_counters.set_bcb_counter(bcb, expression)?; @@ -418,10 +377,10 @@ impl<'a> MakeBcbCounters<'a> { // If the BCB already has a counter, return it. if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] { debug!( - "{}{:?} already has a counter: {}", + "{}{:?} already has a counter: {:?}", NESTED_INDENT.repeat(debug_indent_level), bcb, - self.format_counter(counter_kind), + counter_kind, ); return Ok(counter_kind.as_operand()); } @@ -431,22 +390,22 @@ impl<'a> MakeBcbCounters<'a> { // program results in a tight infinite loop, but it should still compile. let one_path_to_target = self.bcb_has_one_path_to_target(bcb); if one_path_to_target || self.bcb_predecessors(bcb).contains(&bcb) { - let counter_kind = self.coverage_counters.make_counter(|| Some(format!("{bcb:?}"))); + let counter_kind = self.coverage_counters.make_counter(); if one_path_to_target { debug!( - "{}{:?} gets a new counter: {}", + "{}{:?} gets a new counter: {:?}", NESTED_INDENT.repeat(debug_indent_level), bcb, - self.format_counter(&counter_kind), + counter_kind, ); } else { debug!( "{}{:?} has itself as its own predecessor. It can't be part of its own \ - Expression sum, so it will get its own new counter: {}. (Note, the compiled \ + Expression sum, so it will get its own new counter: {:?}. (Note, the compiled \ code will generate an infinite loop.)", NESTED_INDENT.repeat(debug_indent_level), bcb, - self.format_counter(&counter_kind), + counter_kind, ); } return self.coverage_counters.set_bcb_counter(bcb, counter_kind); @@ -481,12 +440,11 @@ impl<'a> MakeBcbCounters<'a> { sumup_edge_counter_operand, Op::Add, edge_counter_operand, - || None, ); debug!( - "{}new intermediate expression: {}", + "{}new intermediate expression: {:?}", NESTED_INDENT.repeat(debug_indent_level), - self.format_counter(&intermediate_expression) + intermediate_expression ); let intermediate_expression_operand = intermediate_expression.as_operand(); self.coverage_counters.intermediate_expressions.push(intermediate_expression); @@ -497,13 +455,12 @@ impl<'a> MakeBcbCounters<'a> { first_edge_counter_operand, Op::Add, some_sumup_edge_counter_operand.unwrap(), - || Some(format!("{bcb:?}")), ); debug!( - "{}{:?} gets a new counter (sum of predecessor counters): {}", + "{}{:?} gets a new counter (sum of predecessor counters): {:?}", NESTED_INDENT.repeat(debug_indent_level), bcb, - self.format_counter(&counter_kind) + counter_kind ); self.coverage_counters.set_bcb_counter(bcb, counter_kind) } @@ -534,24 +491,23 @@ impl<'a> MakeBcbCounters<'a> { self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb)) { debug!( - "{}Edge {:?}->{:?} already has a counter: {}", + "{}Edge {:?}->{:?} already has a counter: {:?}", NESTED_INDENT.repeat(debug_indent_level), from_bcb, to_bcb, - self.format_counter(counter_kind) + counter_kind ); return Ok(counter_kind.as_operand()); } // Make a new counter to count this edge. - let counter_kind = - self.coverage_counters.make_counter(|| Some(format!("{from_bcb:?}->{to_bcb:?}"))); + let counter_kind = self.coverage_counters.make_counter(); debug!( - "{}Edge {:?}->{:?} gets a new counter: {}", + "{}Edge {:?}->{:?} gets a new counter: {:?}", NESTED_INDENT.repeat(debug_indent_level), from_bcb, to_bcb, - self.format_counter(&counter_kind) + counter_kind ); self.coverage_counters.set_bcb_edge_counter(from_bcb, to_bcb, counter_kind) } @@ -710,9 +666,4 @@ impl<'a> MakeBcbCounters<'a> { fn bcb_dominates(&self, dom: BasicCoverageBlock, node: BasicCoverageBlock) -> bool { self.basic_coverage_blocks.dominates(dom, node) } - - #[inline] - fn format_counter(&self, counter_kind: &BcbCounter) -> String { - self.coverage_counters.debug_counters.format_counter(counter_kind) - } } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs deleted file mode 100644 index af616c498fd..00000000000 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ /dev/null @@ -1,802 +0,0 @@ -//! The `InstrumentCoverage` MIR pass implementation includes debugging tools and options -//! to help developers understand and/or improve the analysis and instrumentation of a MIR. -//! -//! To enable coverage, include the rustc command line option: -//! -//! * `-C instrument-coverage` -//! -//! MIR Dump Files, with additional `CoverageGraph` graphviz and `CoverageSpan` spanview -//! ------------------------------------------------------------------------------------ -//! -//! Additional debugging options include: -//! -//! * `-Z dump-mir=InstrumentCoverage` - Generate `.mir` files showing the state of the MIR, -//! before and after the `InstrumentCoverage` pass, for each compiled function. -//! -//! * `-Z dump-mir-graphviz` - If `-Z dump-mir` is also enabled for the current MIR node path, -//! each MIR dump is accompanied by a before-and-after graphical view of the MIR, in Graphviz -//! `.dot` file format (which can be visually rendered as a graph using any of a number of free -//! Graphviz viewers and IDE extensions). -//! -//! For the `InstrumentCoverage` pass, this option also enables generation of an additional -//! Graphviz `.dot` file for each function, rendering the `CoverageGraph`: the control flow -//! graph (CFG) of `BasicCoverageBlocks` (BCBs), as nodes, internally labeled to show the -//! `CoverageSpan`-based MIR elements each BCB represents (`BasicBlock`s, `Statement`s and -//! `Terminator`s), assigned coverage counters and/or expressions, and edge counters, as needed. -//! -//! (Note the additional option, `-Z graphviz-dark-mode`, can be added, to change the rendered -//! output from its default black-on-white background to a dark color theme, if desired.) -//! -//! * `-Z dump-mir-spanview` - If `-Z dump-mir` is also enabled for the current MIR node path, -//! each MIR dump is accompanied by a before-and-after `.html` document showing the function's -//! original source code, highlighted by it's MIR spans, at the `statement`-level (by default), -//! `terminator` only, or encompassing span for the `Terminator` plus all `Statement`s, in each -//! `block` (`BasicBlock`). -//! -//! For the `InstrumentCoverage` pass, this option also enables generation of an additional -//! spanview `.html` file for each function, showing the aggregated `CoverageSpan`s that will -//! require counters (or counter expressions) for accurate coverage analysis. -//! -//! Debug Logging -//! ------------- -//! -//! The `InstrumentCoverage` pass includes debug logging messages at various phases and decision -//! points, which can be enabled via environment variable: -//! -//! ```shell -//! RUSTC_LOG=rustc_mir_transform::transform::coverage=debug -//! ``` -//! -//! Other module paths with coverage-related debug logs may also be of interest, particularly for -//! debugging the coverage map data, injected as global variables in the LLVM IR (during rustc's -//! code generation pass). For example: -//! -//! ```shell -//! RUSTC_LOG=rustc_mir_transform::transform::coverage,rustc_codegen_ssa::coverageinfo,rustc_codegen_llvm::coverageinfo=debug -//! ``` -//! -//! Coverage Debug Options -//! --------------------------------- -//! -//! Additional debugging options can be enabled using the environment variable: -//! -//! ```shell -//! RUSTC_COVERAGE_DEBUG_OPTIONS=<options> -//! ``` -//! -//! These options are comma-separated, and specified in the format `option-name=value`. For example: -//! -//! ```shell -//! $ RUSTC_COVERAGE_DEBUG_OPTIONS=counter-format=id+operation,allow-unused-expressions=yes cargo build -//! ``` -//! -//! Coverage debug options include: -//! -//! * `allow-unused-expressions=yes` or `no` (default: `no`) -//! -//! The `InstrumentCoverage` algorithms _should_ only create and assign expressions to a -//! `BasicCoverageBlock`, or an incoming edge, if that expression is either (a) required to -//! count a `CoverageSpan`, or (b) a dependency of some other required counter expression. -//! -//! If an expression is generated that does not map to a `CoverageSpan` or dependency, this -//! probably indicates there was a bug in the algorithm that creates and assigns counters -//! and expressions. -//! -//! When this kind of bug is encountered, the rustc compiler will panic by default. Setting: -//! `allow-unused-expressions=yes` will log a warning message instead of panicking (effectively -//! ignoring the unused expressions), which may be helpful when debugging the root cause of -//! the problem. -//! -//! * `counter-format=<choices>`, where `<choices>` can be any plus-separated combination of `id`, -//! `block`, and/or `operation` (default: `block+operation`) -//! -//! This option effects both the `CoverageGraph` (graphviz `.dot` files) and debug logging, when -//! generating labels for counters and expressions. -//! -//! Depending on the values and combinations, counters can be labeled by: -//! -//! * `id` - counter or expression ID (ascending counter IDs, starting at 1, or descending -//! expression IDs, starting at `u32:MAX`) -//! * `block` - the `BasicCoverageBlock` label (for example, `bcb0`) or edge label (for -//! example `bcb0->bcb1`), for counters or expressions assigned to count a -//! `BasicCoverageBlock` or edge. Intermediate expressions (not directly associated with -//! a BCB or edge) will be labeled by their expression ID, unless `operation` is also -//! specified. -//! * `operation` - applied to expressions only, labels include the left-hand-side counter -//! or expression label (lhs operand), the operator (`+` or `-`), and the right-hand-side -//! counter or expression (rhs operand). Expression operand labels are generated -//! recursively, generating labels with nested operations, enclosed in parentheses -//! (for example: `bcb2 + (bcb0 - bcb1)`). - -use super::counters::{BcbCounter, CoverageCounters}; -use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph}; -use super::spans::CoverageSpan; - -use itertools::Itertools; -use rustc_middle::mir::create_dump_file; -use rustc_middle::mir::generic_graphviz::GraphvizWriter; -use rustc_middle::mir::spanview::{self, SpanViewable}; - -use rustc_data_structures::fx::FxHashMap; -use rustc_middle::mir::coverage::*; -use rustc_middle::mir::{self, BasicBlock}; -use rustc_middle::ty::TyCtxt; -use rustc_span::Span; - -use std::iter; -use std::ops::Deref; -use std::sync::OnceLock; - -pub const NESTED_INDENT: &str = " "; - -const RUSTC_COVERAGE_DEBUG_OPTIONS: &str = "RUSTC_COVERAGE_DEBUG_OPTIONS"; - -pub(super) fn debug_options<'a>() -> &'a DebugOptions { - static DEBUG_OPTIONS: OnceLock<DebugOptions> = OnceLock::new(); - - &DEBUG_OPTIONS.get_or_init(DebugOptions::from_env) -} - -/// Parses and maintains coverage-specific debug options captured from the environment variable -/// "RUSTC_COVERAGE_DEBUG_OPTIONS", if set. -#[derive(Debug, Clone)] -pub(super) struct DebugOptions { - pub allow_unused_expressions: bool, - counter_format: ExpressionFormat, -} - -impl DebugOptions { - fn from_env() -> Self { - let mut allow_unused_expressions = true; - let mut counter_format = ExpressionFormat::default(); - - if let Ok(env_debug_options) = std::env::var(RUSTC_COVERAGE_DEBUG_OPTIONS) { - for setting_str in env_debug_options.replace(' ', "").replace('-', "_").split(',') { - let (option, value) = match setting_str.split_once('=') { - None => (setting_str, None), - Some((k, v)) => (k, Some(v)), - }; - match option { - "allow_unused_expressions" => { - allow_unused_expressions = bool_option_val(option, value); - debug!( - "{} env option `allow_unused_expressions` is set to {}", - RUSTC_COVERAGE_DEBUG_OPTIONS, allow_unused_expressions - ); - } - "counter_format" => { - match value { - None => { - bug!( - "`{}` option in environment variable {} requires one or more \ - plus-separated choices (a non-empty subset of \ - `id+block+operation`)", - option, - RUSTC_COVERAGE_DEBUG_OPTIONS - ); - } - Some(val) => { - counter_format = counter_format_option_val(val); - debug!( - "{} env option `counter_format` is set to {:?}", - RUSTC_COVERAGE_DEBUG_OPTIONS, counter_format - ); - } - }; - } - _ => bug!( - "Unsupported setting `{}` in environment variable {}", - option, - RUSTC_COVERAGE_DEBUG_OPTIONS - ), - }; - } - } - - Self { allow_unused_expressions, counter_format } - } -} - -fn bool_option_val(option: &str, some_strval: Option<&str>) -> bool { - if let Some(val) = some_strval { - if ["yes", "y", "on", "true"].contains(&val) { - true - } else if ["no", "n", "off", "false"].contains(&val) { - false - } else { - bug!( - "Unsupported value `{}` for option `{}` in environment variable {}", - option, - val, - RUSTC_COVERAGE_DEBUG_OPTIONS - ) - } - } else { - true - } -} - -fn counter_format_option_val(strval: &str) -> ExpressionFormat { - let mut counter_format = ExpressionFormat { id: false, block: false, operation: false }; - let components = strval.splitn(3, '+'); - for component in components { - match component { - "id" => counter_format.id = true, - "block" => counter_format.block = true, - "operation" => counter_format.operation = true, - _ => bug!( - "Unsupported counter_format choice `{}` in environment variable {}", - component, - RUSTC_COVERAGE_DEBUG_OPTIONS - ), - } - } - counter_format -} - -#[derive(Debug, Clone)] -struct ExpressionFormat { - id: bool, - block: bool, - operation: bool, -} - -impl Default for ExpressionFormat { - fn default() -> Self { - Self { id: false, block: true, operation: true } - } -} - -/// If enabled, this struct maintains a map from `BcbCounter` IDs (as `Operand`) to -/// the `BcbCounter` data and optional label (normally, the counter's associated -/// `BasicCoverageBlock` format string, if any). -/// -/// Use `format_counter` to convert one of these `BcbCounter` counters to a debug output string, -/// as directed by the `DebugOptions`. This allows the format of counter labels in logs and dump -/// files (including the `CoverageGraph` graphviz file) to be changed at runtime, via environment -/// variable. -/// -/// `DebugCounters` supports a recursive rendering of `Expression` counters, so they can be -/// presented as nested expressions such as `(bcb3 - (bcb0 + bcb1))`. -pub(super) struct DebugCounters { - some_counters: Option<FxHashMap<Operand, DebugCounter>>, -} - -impl DebugCounters { - pub fn new() -> Self { - Self { some_counters: None } - } - - pub fn enable(&mut self) { - debug_assert!(!self.is_enabled()); - self.some_counters.replace(FxHashMap::default()); - } - - pub fn is_enabled(&self) -> bool { - self.some_counters.is_some() - } - - pub fn add_counter(&mut self, counter_kind: &BcbCounter, some_block_label: Option<String>) { - if let Some(counters) = &mut self.some_counters { - let id = counter_kind.as_operand(); - counters - .try_insert(id, DebugCounter::new(counter_kind.clone(), some_block_label)) - .expect("attempt to add the same counter_kind to DebugCounters more than once"); - } - } - - pub fn some_block_label(&self, operand: Operand) -> Option<&String> { - self.some_counters.as_ref().and_then(|counters| { - counters.get(&operand).and_then(|debug_counter| debug_counter.some_block_label.as_ref()) - }) - } - - pub fn format_counter(&self, counter_kind: &BcbCounter) -> String { - match *counter_kind { - BcbCounter::Counter { .. } => { - format!("Counter({})", self.format_counter_kind(counter_kind)) - } - BcbCounter::Expression { .. } => { - format!("Expression({})", self.format_counter_kind(counter_kind)) - } - } - } - - fn format_counter_kind(&self, counter_kind: &BcbCounter) -> String { - let counter_format = &debug_options().counter_format; - if let BcbCounter::Expression { id, lhs, op, rhs } = *counter_kind { - if counter_format.operation { - return format!( - "{}{} {} {}", - if counter_format.id || self.some_counters.is_none() { - format!("#{} = ", id.index()) - } else { - String::new() - }, - self.format_operand(lhs), - match op { - Op::Add => "+", - Op::Subtract => "-", - }, - self.format_operand(rhs), - ); - } - } - - let id = counter_kind.as_operand(); - if self.some_counters.is_some() && (counter_format.block || !counter_format.id) { - let counters = self.some_counters.as_ref().unwrap(); - if let Some(DebugCounter { some_block_label: Some(block_label), .. }) = - counters.get(&id) - { - return if counter_format.id { - format!("{}#{:?}", block_label, id) - } else { - block_label.to_string() - }; - } - } - format!("#{:?}", id) - } - - fn format_operand(&self, operand: Operand) -> String { - if matches!(operand, Operand::Zero) { - return String::from("0"); - } - if let Some(counters) = &self.some_counters { - if let Some(DebugCounter { counter_kind, some_block_label }) = counters.get(&operand) { - if let BcbCounter::Expression { .. } = counter_kind { - if let Some(label) = some_block_label && debug_options().counter_format.block { - return format!( - "{}:({})", - label, - self.format_counter_kind(counter_kind) - ); - } - return format!("({})", self.format_counter_kind(counter_kind)); - } - return self.format_counter_kind(counter_kind); - } - } - format!("#{:?}", operand) - } -} - -/// A non-public support class to `DebugCounters`. -#[derive(Debug)] -struct DebugCounter { - counter_kind: BcbCounter, - some_block_label: Option<String>, -} - -impl DebugCounter { - fn new(counter_kind: BcbCounter, some_block_label: Option<String>) -> Self { - Self { counter_kind, some_block_label } - } -} - -/// If enabled, this data structure captures additional debugging information used when generating -/// a Graphviz (.dot file) representation of the `CoverageGraph`, for debugging purposes. -pub(super) struct GraphvizData { - some_bcb_to_coverage_spans_with_counters: - Option<FxHashMap<BasicCoverageBlock, Vec<(CoverageSpan, BcbCounter)>>>, - some_bcb_to_dependency_counters: Option<FxHashMap<BasicCoverageBlock, Vec<BcbCounter>>>, - some_edge_to_counter: Option<FxHashMap<(BasicCoverageBlock, BasicBlock), BcbCounter>>, -} - -impl GraphvizData { - pub fn new() -> Self { - Self { - some_bcb_to_coverage_spans_with_counters: None, - some_bcb_to_dependency_counters: None, - some_edge_to_counter: None, - } - } - - pub fn enable(&mut self) { - debug_assert!(!self.is_enabled()); - self.some_bcb_to_coverage_spans_with_counters = Some(FxHashMap::default()); - self.some_bcb_to_dependency_counters = Some(FxHashMap::default()); - self.some_edge_to_counter = Some(FxHashMap::default()); - } - - pub fn is_enabled(&self) -> bool { - self.some_bcb_to_coverage_spans_with_counters.is_some() - } - - pub fn add_bcb_coverage_span_with_counter( - &mut self, - bcb: BasicCoverageBlock, - coverage_span: &CoverageSpan, - counter_kind: &BcbCounter, - ) { - if let Some(bcb_to_coverage_spans_with_counters) = - self.some_bcb_to_coverage_spans_with_counters.as_mut() - { - bcb_to_coverage_spans_with_counters - .entry(bcb) - .or_insert_with(Vec::new) - .push((coverage_span.clone(), counter_kind.clone())); - } - } - - pub fn get_bcb_coverage_spans_with_counters( - &self, - bcb: BasicCoverageBlock, - ) -> Option<&[(CoverageSpan, BcbCounter)]> { - if let Some(bcb_to_coverage_spans_with_counters) = - self.some_bcb_to_coverage_spans_with_counters.as_ref() - { - bcb_to_coverage_spans_with_counters.get(&bcb).map(Deref::deref) - } else { - None - } - } - - pub fn add_bcb_dependency_counter( - &mut self, - bcb: BasicCoverageBlock, - counter_kind: &BcbCounter, - ) { - if let Some(bcb_to_dependency_counters) = self.some_bcb_to_dependency_counters.as_mut() { - bcb_to_dependency_counters - .entry(bcb) - .or_insert_with(Vec::new) - .push(counter_kind.clone()); - } - } - - pub fn get_bcb_dependency_counters(&self, bcb: BasicCoverageBlock) -> Option<&[BcbCounter]> { - if let Some(bcb_to_dependency_counters) = self.some_bcb_to_dependency_counters.as_ref() { - bcb_to_dependency_counters.get(&bcb).map(Deref::deref) - } else { - None - } - } - - pub fn set_edge_counter( - &mut self, - from_bcb: BasicCoverageBlock, - to_bb: BasicBlock, - counter_kind: &BcbCounter, - ) { - if let Some(edge_to_counter) = self.some_edge_to_counter.as_mut() { - edge_to_counter - .try_insert((from_bcb, to_bb), counter_kind.clone()) - .expect("invalid attempt to insert more than one edge counter for the same edge"); - } - } - - pub fn get_edge_counter( - &self, - from_bcb: BasicCoverageBlock, - to_bb: BasicBlock, - ) -> Option<&BcbCounter> { - if let Some(edge_to_counter) = self.some_edge_to_counter.as_ref() { - edge_to_counter.get(&(from_bcb, to_bb)) - } else { - None - } - } -} - -/// If enabled, this struct captures additional data used to track whether expressions were used, -/// directly or indirectly, to compute the coverage counts for all `CoverageSpan`s, and any that are -/// _not_ used are retained in the `unused_expressions` Vec, to be included in debug output (logs -/// and/or a `CoverageGraph` graphviz output). -pub(super) struct UsedExpressions { - some_used_expression_operands: Option<FxHashMap<Operand, Vec<ExpressionId>>>, - some_unused_expressions: - Option<Vec<(BcbCounter, Option<BasicCoverageBlock>, BasicCoverageBlock)>>, -} - -impl UsedExpressions { - pub fn new() -> Self { - Self { some_used_expression_operands: None, some_unused_expressions: None } - } - - pub fn enable(&mut self) { - debug_assert!(!self.is_enabled()); - self.some_used_expression_operands = Some(FxHashMap::default()); - self.some_unused_expressions = Some(Vec::new()); - } - - pub fn is_enabled(&self) -> bool { - self.some_used_expression_operands.is_some() - } - - pub fn add_expression_operands(&mut self, expression: &BcbCounter) { - if let Some(used_expression_operands) = self.some_used_expression_operands.as_mut() { - if let BcbCounter::Expression { id, lhs, rhs, .. } = *expression { - used_expression_operands.entry(lhs).or_insert_with(Vec::new).push(id); - used_expression_operands.entry(rhs).or_insert_with(Vec::new).push(id); - } - } - } - - pub fn expression_is_used(&self, expression: &BcbCounter) -> bool { - if let Some(used_expression_operands) = self.some_used_expression_operands.as_ref() { - used_expression_operands.contains_key(&expression.as_operand()) - } else { - false - } - } - - pub fn add_unused_expression_if_not_found( - &mut self, - expression: &BcbCounter, - edge_from_bcb: Option<BasicCoverageBlock>, - target_bcb: BasicCoverageBlock, - ) { - if let Some(used_expression_operands) = self.some_used_expression_operands.as_ref() { - if !used_expression_operands.contains_key(&expression.as_operand()) { - self.some_unused_expressions.as_mut().unwrap().push(( - expression.clone(), - edge_from_bcb, - target_bcb, - )); - } - } - } - - /// Return the list of unused counters (if any) as a tuple with the counter (`BcbCounter`), - /// optional `from_bcb` (if it was an edge counter), and `target_bcb`. - pub fn get_unused_expressions( - &self, - ) -> Vec<(BcbCounter, Option<BasicCoverageBlock>, BasicCoverageBlock)> { - if let Some(unused_expressions) = self.some_unused_expressions.as_ref() { - unused_expressions.clone() - } else { - Vec::new() - } - } - - /// If enabled, validate that every BCB or edge counter not directly associated with a coverage - /// span is at least indirectly associated (it is a dependency of a BCB counter that _is_ - /// associated with a coverage span). - pub fn validate( - &mut self, - bcb_counters_without_direct_coverage_spans: &[( - Option<BasicCoverageBlock>, - BasicCoverageBlock, - BcbCounter, - )], - ) { - if self.is_enabled() { - let mut not_validated = bcb_counters_without_direct_coverage_spans - .iter() - .map(|(_, _, counter_kind)| counter_kind) - .collect::<Vec<_>>(); - let mut validating_count = 0; - while not_validated.len() != validating_count { - let to_validate = not_validated.split_off(0); - validating_count = to_validate.len(); - for counter_kind in to_validate { - if self.expression_is_used(counter_kind) { - self.add_expression_operands(counter_kind); - } else { - not_validated.push(counter_kind); - } - } - } - } - } - - pub fn alert_on_unused_expressions(&self, debug_counters: &DebugCounters) { - if let Some(unused_expressions) = self.some_unused_expressions.as_ref() { - for (counter_kind, edge_from_bcb, target_bcb) in unused_expressions { - let unused_counter_message = if let Some(from_bcb) = edge_from_bcb.as_ref() { - format!( - "non-coverage edge counter found without a dependent expression, in \ - {:?}->{:?}; counter={}", - from_bcb, - target_bcb, - debug_counters.format_counter(&counter_kind), - ) - } else { - format!( - "non-coverage counter found without a dependent expression, in {:?}; \ - counter={}", - target_bcb, - debug_counters.format_counter(&counter_kind), - ) - }; - - if debug_options().allow_unused_expressions { - debug!("WARNING: {}", unused_counter_message); - } else { - bug!("{}", unused_counter_message); - } - } - } - } -} - -/// Generates the MIR pass `CoverageSpan`-specific spanview dump file. -pub(super) fn dump_coverage_spanview<'tcx>( - tcx: TyCtxt<'tcx>, - mir_body: &mir::Body<'tcx>, - basic_coverage_blocks: &CoverageGraph, - pass_name: &str, - body_span: Span, - coverage_spans: &[CoverageSpan], -) { - let mir_source = mir_body.source; - let def_id = mir_source.def_id(); - - let span_viewables = span_viewables(tcx, mir_body, basic_coverage_blocks, &coverage_spans); - let mut file = create_dump_file(tcx, "html", false, pass_name, &0i32, mir_body) - .expect("Unexpected error creating MIR spanview HTML file"); - let crate_name = tcx.crate_name(def_id.krate); - let item_name = tcx.def_path(def_id).to_filename_friendly_no_crate(); - let title = format!("{crate_name}.{item_name} - Coverage Spans"); - spanview::write_document(tcx, body_span, span_viewables, &title, &mut file) - .expect("Unexpected IO error dumping coverage spans as HTML"); -} - -/// Converts the computed `BasicCoverageBlockData`s into `SpanViewable`s. -fn span_viewables<'tcx>( - tcx: TyCtxt<'tcx>, - mir_body: &mir::Body<'tcx>, - basic_coverage_blocks: &CoverageGraph, - coverage_spans: &[CoverageSpan], -) -> Vec<SpanViewable> { - let mut span_viewables = Vec::new(); - for coverage_span in coverage_spans { - let tooltip = coverage_span.format_coverage_statements(tcx, mir_body); - let CoverageSpan { span, bcb, .. } = coverage_span; - let bcb_data = &basic_coverage_blocks[*bcb]; - let id = bcb_data.id(); - let leader_bb = bcb_data.leader_bb(); - span_viewables.push(SpanViewable { bb: leader_bb, span: *span, id, tooltip }); - } - span_viewables -} - -/// Generates the MIR pass coverage-specific graphviz dump file. -pub(super) fn dump_coverage_graphviz<'tcx>( - tcx: TyCtxt<'tcx>, - mir_body: &mir::Body<'tcx>, - pass_name: &str, - basic_coverage_blocks: &CoverageGraph, - coverage_counters: &CoverageCounters, - graphviz_data: &GraphvizData, - intermediate_expressions: &[BcbCounter], - debug_used_expressions: &UsedExpressions, -) { - let debug_counters = &coverage_counters.debug_counters; - - let mir_source = mir_body.source; - let def_id = mir_source.def_id(); - let node_content = |bcb| { - bcb_to_string_sections( - tcx, - mir_body, - coverage_counters, - bcb, - &basic_coverage_blocks[bcb], - graphviz_data.get_bcb_coverage_spans_with_counters(bcb), - graphviz_data.get_bcb_dependency_counters(bcb), - // intermediate_expressions are injected into the mir::START_BLOCK, so - // include them in the first BCB. - if bcb.index() == 0 { Some(&intermediate_expressions) } else { None }, - ) - }; - let edge_labels = |from_bcb| { - let from_bcb_data = &basic_coverage_blocks[from_bcb]; - let from_terminator = from_bcb_data.terminator(mir_body); - let mut edge_labels = from_terminator.kind.fmt_successor_labels(); - edge_labels.retain(|label| label != "unreachable"); - let edge_counters = from_terminator - .successors() - .map(|successor_bb| graphviz_data.get_edge_counter(from_bcb, successor_bb)); - iter::zip(&edge_labels, edge_counters) - .map(|(label, some_counter)| { - if let Some(counter) = some_counter { - format!("{}\n{}", label, debug_counters.format_counter(counter)) - } else { - label.to_string() - } - }) - .collect::<Vec<_>>() - }; - let graphviz_name = format!("Cov_{}_{}", def_id.krate.index(), def_id.index.index()); - let mut graphviz_writer = - GraphvizWriter::new(basic_coverage_blocks, &graphviz_name, node_content, edge_labels); - let unused_expressions = debug_used_expressions.get_unused_expressions(); - if unused_expressions.len() > 0 { - graphviz_writer.set_graph_label(&format!( - "Unused expressions:\n {}", - unused_expressions - .as_slice() - .iter() - .map(|(counter_kind, edge_from_bcb, target_bcb)| { - if let Some(from_bcb) = edge_from_bcb.as_ref() { - format!( - "{:?}->{:?}: {}", - from_bcb, - target_bcb, - debug_counters.format_counter(&counter_kind), - ) - } else { - format!( - "{:?}: {}", - target_bcb, - debug_counters.format_counter(&counter_kind), - ) - } - }) - .join("\n ") - )); - } - let mut file = create_dump_file(tcx, "dot", false, pass_name, &0i32, mir_body) - .expect("Unexpected error creating BasicCoverageBlock graphviz DOT file"); - graphviz_writer - .write_graphviz(tcx, &mut file) - .expect("Unexpected error writing BasicCoverageBlock graphviz DOT file"); -} - -fn bcb_to_string_sections<'tcx>( - tcx: TyCtxt<'tcx>, - mir_body: &mir::Body<'tcx>, - coverage_counters: &CoverageCounters, - bcb: BasicCoverageBlock, - bcb_data: &BasicCoverageBlockData, - some_coverage_spans_with_counters: Option<&[(CoverageSpan, BcbCounter)]>, - some_dependency_counters: Option<&[BcbCounter]>, - some_intermediate_expressions: Option<&[BcbCounter]>, -) -> Vec<String> { - let debug_counters = &coverage_counters.debug_counters; - - let len = bcb_data.basic_blocks.len(); - let mut sections = Vec::new(); - if let Some(collect_intermediate_expressions) = some_intermediate_expressions { - sections.push( - collect_intermediate_expressions - .iter() - .map(|expression| { - format!("Intermediate {}", debug_counters.format_counter(expression)) - }) - .join("\n"), - ); - } - if let Some(coverage_spans_with_counters) = some_coverage_spans_with_counters { - sections.push( - coverage_spans_with_counters - .iter() - .map(|(covspan, counter)| { - format!( - "{} at {}", - debug_counters.format_counter(counter), - covspan.format(tcx, mir_body) - ) - }) - .join("\n"), - ); - } - if let Some(dependency_counters) = some_dependency_counters { - sections.push(format!( - "Non-coverage counters:\n {}", - dependency_counters - .iter() - .map(|counter| debug_counters.format_counter(counter)) - .join(" \n"), - )); - } - if let Some(counter_kind) = coverage_counters.bcb_counter(bcb) { - sections.push(format!("{counter_kind:?}")); - } - let non_term_blocks = bcb_data.basic_blocks[0..len - 1] - .iter() - .map(|&bb| format!("{:?}: {}", bb, mir_body[bb].terminator().kind.name())) - .collect::<Vec<_>>(); - if non_term_blocks.len() > 0 { - sections.push(non_term_blocks.join("\n")); - } - sections.push(format!( - "{:?}: {}", - bcb_data.basic_blocks.last().unwrap(), - bcb_data.terminator(mir_body).kind.name(), - )); - sections -} diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 60461691e7f..ff2254d6941 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -1,4 +1,3 @@ -use itertools::Itertools; use rustc_data_structures::graph::dominators::{self, Dominators}; use rustc_data_structures::graph::{self, GraphSuccessors, WithNumNodes, WithStartNode}; use rustc_index::bit_set::BitSet; @@ -8,8 +7,6 @@ use rustc_middle::mir::{self, BasicBlock, BasicBlockData, Terminator, Terminator use std::cmp::Ordering; use std::ops::{Index, IndexMut}; -const ID_SEPARATOR: &str = ","; - /// A coverage-specific simplification of the MIR control flow graph (CFG). The `CoverageGraph`s /// nodes are `BasicCoverageBlock`s, which encompass one or more MIR `BasicBlock`s. #[derive(Debug)] @@ -199,12 +196,8 @@ impl CoverageGraph { } #[inline(always)] - pub fn rank_partial_cmp( - &self, - a: BasicCoverageBlock, - b: BasicCoverageBlock, - ) -> Option<Ordering> { - self.dominators.as_ref().unwrap().rank_partial_cmp(a, b) + pub fn cmp_in_dominator_order(&self, a: BasicCoverageBlock, b: BasicCoverageBlock) -> Ordering { + self.dominators.as_ref().unwrap().cmp_in_dominator_order(a, b) } } @@ -328,10 +321,6 @@ impl BasicCoverageBlockData { pub fn terminator<'a, 'tcx>(&self, mir_body: &'a mir::Body<'tcx>) -> &'a Terminator<'tcx> { &mir_body[self.last_bb()].terminator() } - - pub fn id(&self) -> String { - format!("@{}", self.basic_blocks.iter().map(|bb| bb.index().to_string()).join(ID_SEPARATOR)) - } } /// Represents a successor from a branching BasicCoverageBlock (such as the arms of a `SwitchInt`) diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index d0b28eb2f5d..c75d33eeb31 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -1,7 +1,6 @@ pub mod query; mod counters; -mod debug; mod graph; mod spans; @@ -20,7 +19,6 @@ use rustc_index::IndexVec; use rustc_middle::hir; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::*; -use rustc_middle::mir::dump_enabled; use rustc_middle::mir::{ self, BasicBlock, BasicBlockData, Coverage, SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, @@ -94,13 +92,12 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage { } trace!("InstrumentCoverage starting for {:?}", mir_source.def_id()); - Instrumentor::new(&self.name(), tcx, mir_body).inject_counters(); + Instrumentor::new(tcx, mir_body).inject_counters(); trace!("InstrumentCoverage done for {:?}", mir_source.def_id()); } } struct Instrumentor<'a, 'tcx> { - pass_name: &'a str, tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>, source_file: Lrc<SourceFile>, @@ -112,7 +109,7 @@ struct Instrumentor<'a, 'tcx> { } impl<'a, 'tcx> Instrumentor<'a, 'tcx> { - fn new(pass_name: &'a str, tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self { + fn new(tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self { let source_map = tcx.sess.source_map(); let def_id = mir_body.source.def_id(); let (some_fn_sig, hir_body) = fn_sig_and_body(tcx, def_id); @@ -141,7 +138,6 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let coverage_counters = CoverageCounters::new(&basic_coverage_blocks); Self { - pass_name, tcx, mir_body, source_file, @@ -154,28 +150,9 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } fn inject_counters(&'a mut self) { - let tcx = self.tcx; - let mir_source = self.mir_body.source; - let def_id = mir_source.def_id(); let fn_sig_span = self.fn_sig_span; let body_span = self.body_span; - let mut graphviz_data = debug::GraphvizData::new(); - let mut debug_used_expressions = debug::UsedExpressions::new(); - - let dump_mir = dump_enabled(tcx, self.pass_name, def_id); - let dump_graphviz = dump_mir && tcx.sess.opts.unstable_opts.dump_mir_graphviz; - let dump_spanview = dump_mir && tcx.sess.opts.unstable_opts.dump_mir_spanview.is_some(); - - if dump_graphviz { - graphviz_data.enable(); - self.coverage_counters.enable_debug(); - } - - if dump_graphviz || level_enabled!(tracing::Level::DEBUG) { - debug_used_expressions.enable(); - } - //////////////////////////////////////////////////// // Compute `CoverageSpan`s from the `CoverageGraph`. let coverage_spans = CoverageSpans::generate_coverage_spans( @@ -185,17 +162,6 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { &self.basic_coverage_blocks, ); - if dump_spanview { - debug::dump_coverage_spanview( - tcx, - self.mir_body, - &self.basic_coverage_blocks, - self.pass_name, - body_span, - &coverage_spans, - ); - } - //////////////////////////////////////////////////// // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure // every `CoverageSpan` has a `Counter` or `Expression` assigned to its `BasicCoverageBlock` @@ -209,14 +175,6 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { .make_bcb_counters(&mut self.basic_coverage_blocks, &coverage_spans); if let Ok(()) = result { - // If debugging, add any intermediate expressions (which are not associated with any - // BCB) to the `debug_used_expressions` map. - if debug_used_expressions.is_enabled() { - for intermediate_expression in &self.coverage_counters.intermediate_expressions { - debug_used_expressions.add_expression_operands(intermediate_expression); - } - } - //////////////////////////////////////////////////// // Remove the counter or edge counter from of each `CoverageSpan`s associated // `BasicCoverageBlock`, and inject a `Coverage` statement into the MIR. @@ -227,11 +185,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { // These `CoverageSpan`-associated counters are removed from their associated // `BasicCoverageBlock`s so that the only remaining counters in the `CoverageGraph` // are indirect counters (to be injected next, without associated code regions). - self.inject_coverage_span_counters( - coverage_spans, - &mut graphviz_data, - &mut debug_used_expressions, - ); + self.inject_coverage_span_counters(coverage_spans); //////////////////////////////////////////////////// // For any remaining `BasicCoverageBlock` counters (that were not associated with @@ -239,37 +193,17 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { // to ensure `BasicCoverageBlock` counters that other `Expression`s may depend on // are in fact counted, even though they don't directly contribute to counting // their own independent code region's coverage. - self.inject_indirect_counters(&mut graphviz_data, &mut debug_used_expressions); + self.inject_indirect_counters(); // Intermediate expressions will be injected as the final step, after generating // debug output, if any. //////////////////////////////////////////////////// }; - if graphviz_data.is_enabled() { - // Even if there was an error, a partial CoverageGraph can still generate a useful - // graphviz output. - debug::dump_coverage_graphviz( - tcx, - self.mir_body, - self.pass_name, - &self.basic_coverage_blocks, - &self.coverage_counters, - &graphviz_data, - &self.coverage_counters.intermediate_expressions, - &debug_used_expressions, - ); - } - if let Err(e) = result { bug!("Error processing: {:?}: {:?}", self.mir_body.source.def_id(), e.message) }; - // Depending on current `debug_options()`, `alert_on_unused_expressions()` could panic, so - // this check is performed as late as possible, to allow other debug output (logs and dump - // files), which might be helpful in analyzing unused expressions, to still be generated. - debug_used_expressions.alert_on_unused_expressions(&self.coverage_counters.debug_counters); - //////////////////////////////////////////////////// // Finally, inject the intermediate expressions collected along the way. for intermediate_expression in &self.coverage_counters.intermediate_expressions { @@ -285,15 +219,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { /// `bcb` to its `Counter`, when injected. Subsequent `CoverageSpan`s for a BCB that already has /// a `Counter` will inject an `Expression` instead, and compute its value by adding `ZERO` to /// the BCB `Counter` value. - /// - /// If debugging, add every BCB `Expression` associated with a `CoverageSpan`s to the - /// `used_expression_operands` map. - fn inject_coverage_span_counters( - &mut self, - coverage_spans: Vec<CoverageSpan>, - graphviz_data: &mut debug::GraphvizData, - debug_used_expressions: &mut debug::UsedExpressions, - ) { + fn inject_coverage_span_counters(&mut self, coverage_spans: Vec<CoverageSpan>) { let tcx = self.tcx; let source_map = tcx.sess.source_map(); let body_span = self.body_span; @@ -307,12 +233,10 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { self.coverage_counters.make_identity_counter(counter_operand) } else if let Some(counter_kind) = self.coverage_counters.take_bcb_counter(bcb) { bcb_counters[bcb] = Some(counter_kind.as_operand()); - debug_used_expressions.add_expression_operands(&counter_kind); counter_kind } else { bug!("Every BasicCoverageBlock should have a Counter or Expression"); }; - graphviz_data.add_bcb_coverage_span_with_counter(bcb, &covspan, &counter_kind); let code_region = make_code_region(source_map, file_name, span, body_span); @@ -333,11 +257,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { /// associated with a `CoverageSpan`, should only exist if the counter is an `Expression` /// dependency (one of the expression operands). Collect them, and inject the additional /// counters into the MIR, without a reportable coverage span. - fn inject_indirect_counters( - &mut self, - graphviz_data: &mut debug::GraphvizData, - debug_used_expressions: &mut debug::UsedExpressions, - ) { + fn inject_indirect_counters(&mut self) { let mut bcb_counters_without_direct_coverage_spans = Vec::new(); for (target_bcb, counter_kind) in self.coverage_counters.drain_bcb_counters() { bcb_counters_without_direct_coverage_spans.push((None, target_bcb, counter_kind)); @@ -352,19 +272,8 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { )); } - // If debug is enabled, validate that every BCB or edge counter not directly associated - // with a coverage span is at least indirectly associated (it is a dependency of a BCB - // counter that _is_ associated with a coverage span). - debug_used_expressions.validate(&bcb_counters_without_direct_coverage_spans); - for (edge_from_bcb, target_bcb, counter_kind) in bcb_counters_without_direct_coverage_spans { - debug_used_expressions.add_unused_expression_if_not_found( - &counter_kind, - edge_from_bcb, - target_bcb, - ); - match counter_kind { BcbCounter::Counter { .. } => { let inject_to_bb = if let Some(from_bcb) = edge_from_bcb { @@ -375,26 +284,17 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let to_bb = self.bcb_leader_bb(target_bcb); let new_bb = inject_edge_counter_basic_block(self.mir_body, from_bb, to_bb); - graphviz_data.set_edge_counter(from_bcb, new_bb, &counter_kind); debug!( "Edge {:?} (last {:?}) -> {:?} (leader {:?}) requires a new MIR \ - BasicBlock {:?}, for unclaimed edge counter {}", - edge_from_bcb, - from_bb, - target_bcb, - to_bb, - new_bb, - self.format_counter(&counter_kind), + BasicBlock {:?}, for unclaimed edge counter {:?}", + edge_from_bcb, from_bb, target_bcb, to_bb, new_bb, counter_kind, ); new_bb } else { let target_bb = self.bcb_last_bb(target_bcb); - graphviz_data.add_bcb_dependency_counter(target_bcb, &counter_kind); debug!( - "{:?} ({:?}) gets a new Coverage statement for unclaimed counter {}", - target_bcb, - target_bb, - self.format_counter(&counter_kind), + "{:?} ({:?}) gets a new Coverage statement for unclaimed counter {:?}", + target_bcb, target_bb, counter_kind, ); target_bb }; @@ -429,11 +329,6 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { &self.basic_coverage_blocks[bcb] } - #[inline] - fn format_counter(&self, counter_kind: &BcbCounter) -> String { - self.coverage_counters.debug_counters.format_counter(counter_kind) - } - fn make_mir_coverage_kind(&self, counter_kind: &BcbCounter) -> CoverageKind { match *counter_kind { BcbCounter::Counter { id } => { diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 717763a94a0..767f8e9f4fa 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,18 +1,14 @@ use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB}; -use itertools::Itertools; use rustc_data_structures::graph::WithNumNodes; -use rustc_middle::mir::spanview::source_range_no_file; use rustc_middle::mir::{ self, AggregateKind, BasicBlock, FakeReadCause, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, }; -use rustc_middle::ty::TyCtxt; use rustc_span::source_map::original_sp; use rustc_span::{BytePos, ExpnKind, MacroKind, Span, Symbol}; use std::cell::OnceCell; -use std::cmp::Ordering; #[derive(Debug, Copy, Clone)] pub(super) enum CoverageStatement { @@ -21,31 +17,6 @@ pub(super) enum CoverageStatement { } impl CoverageStatement { - pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { - match *self { - Self::Statement(bb, span, stmt_index) => { - let stmt = &mir_body[bb].statements[stmt_index]; - format!( - "{}: @{}[{}]: {:?}", - source_range_no_file(tcx, span), - bb.index(), - stmt_index, - stmt - ) - } - Self::Terminator(bb, span) => { - let term = mir_body[bb].terminator(); - format!( - "{}: @{}.{}: {:?}", - source_range_no_file(tcx, span), - bb.index(), - term.kind.name(), - term.kind - ) - } - } - } - pub fn span(&self) -> Span { match self { Self::Statement(_, span, _) | Self::Terminator(_, span) => *span, @@ -151,27 +122,6 @@ impl CoverageSpan { self.bcb == other.bcb } - pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { - format!( - "{}\n {}", - source_range_no_file(tcx, self.span), - self.format_coverage_statements(tcx, mir_body).replace('\n', "\n "), - ) - } - - pub fn format_coverage_statements<'tcx>( - &self, - tcx: TyCtxt<'tcx>, - mir_body: &mir::Body<'tcx>, - ) -> String { - let mut sorted_coverage_statements = self.coverage_statements.clone(); - sorted_coverage_statements.sort_unstable_by_key(|covstmt| match *covstmt { - CoverageStatement::Statement(bb, _, index) => (bb, index), - CoverageStatement::Terminator(bb, _) => (bb, usize::MAX), - }); - sorted_coverage_statements.iter().map(|covstmt| covstmt.format(tcx, mir_body)).join("\n") - } - /// If the span is part of a macro, returns the macro name symbol. pub fn current_macro(&self) -> Option<Symbol> { self.current_macro_or_none @@ -333,30 +283,21 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { initial_spans.push(CoverageSpan::for_fn_sig(self.fn_sig_span)); - initial_spans.sort_unstable_by(|a, b| { - if a.span.lo() == b.span.lo() { - if a.span.hi() == b.span.hi() { - if a.is_in_same_bcb(b) { - Some(Ordering::Equal) - } else { - // Sort equal spans by dominator relationship (so dominators always come - // before the dominated equal spans). When later comparing two spans in - // order, the first will either dominate the second, or they will have no - // dominator relationship. - self.basic_coverage_blocks.rank_partial_cmp(a.bcb, b.bcb) - } - } else { - // Sort hi() in reverse order so shorter spans are attempted after longer spans. - // This guarantees that, if a `prev` span overlaps, and is not equal to, a - // `curr` span, the prev span either extends further left of the curr span, or - // they start at the same position and the prev span extends further right of - // the end of the curr span. - b.span.hi().partial_cmp(&a.span.hi()) - } - } else { - a.span.lo().partial_cmp(&b.span.lo()) - } - .unwrap() + initial_spans.sort_by(|a, b| { + // First sort by span start. + Ord::cmp(&a.span.lo(), &b.span.lo()) + // If span starts are the same, sort by span end in reverse order. + // This ensures that if spans A and B are adjacent in the list, + // and they overlap but are not equal, then either: + // - Span A extends further left, or + // - Both have the same start and span A extends further right + .then_with(|| Ord::cmp(&a.span.hi(), &b.span.hi()).reverse()) + // If both spans are equal, sort the BCBs in dominator order, + // so that dominating BCBs come before other BCBs they dominate. + .then_with(|| self.basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb)) + // If two spans are otherwise identical, put closure spans first, + // as this seems to be what the refinement step expects. + .then_with(|| Ord::cmp(&a.is_closure, &b.is_closure).reverse()) }); initial_spans @@ -822,7 +763,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> // and `_1` is the `Place` for `somenum`. // // If and when the Issue is resolved, remove this special case match pattern: - StatementKind::FakeRead(box (cause, _)) if cause == FakeReadCause::ForGuardBinding => None, + StatementKind::FakeRead(box (FakeReadCause::ForGuardBinding, _)) => None, // Retain spans from all other statements StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding` diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 333a14be996..7b14fef6153 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -3,17 +3,19 @@ //! Currently, this pass only propagates scalar values. use rustc_const_eval::const_eval::CheckAlignment; -use rustc_const_eval::interpret::{ConstValue, ImmTy, Immediate, InterpCx, Scalar}; +use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; use rustc_data_structures::fx::FxHashMap; use rustc_hir::def::DefKind; -use rustc_middle::mir::visit::{MutVisitor, NonMutatingUseContext, PlaceContext, Visitor}; +use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult, Scalar}; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{ - Map, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace, + Map, PlaceIndex, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace, }; use rustc_mir_dataflow::{lattice::FlatSet, Analysis, Results, ResultsVisitor}; +use rustc_span::def_id::DefId; use rustc_span::DUMMY_SP; use rustc_target::abi::{Align, FieldIdx, VariantIdx}; @@ -58,13 +60,10 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp { .in_scope(|| analysis.wrap().into_engine(tcx, body).iterate_to_fixpoint()); // Collect results and patch the body afterwards. - let mut visitor = CollectAndPatch::new(tcx, &body.local_decls); + let mut visitor = Collector::new(tcx, &body.local_decls); debug_span!("collect").in_scope(|| results.visit_reachable_with(body, &mut visitor)); - debug_span!("patch").in_scope(|| { - for (block, bbdata) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { - visitor.visit_basic_block_data(block, bbdata); - } - }) + let mut patch = visitor.patch; + debug_span!("patch").in_scope(|| patch.visit_body_preserves_cfg(body)); } } @@ -77,7 +76,7 @@ struct ConstAnalysis<'a, 'tcx> { } impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { - type Value = FlatSet<ScalarInt>; + type Value = FlatSet<Scalar>; const NAME: &'static str = "ConstAnalysis"; @@ -111,6 +110,18 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { state: &mut State<Self::Value>, ) { match rvalue { + Rvalue::Use(operand) => { + state.flood(target.as_ref(), self.map()); + if let Some(target) = self.map.find(target.as_ref()) { + self.assign_operand(state, target, operand); + } + } + Rvalue::CopyForDeref(rhs) => { + state.flood(target.as_ref(), self.map()); + if let Some(target) = self.map.find(target.as_ref()) { + self.assign_operand(state, target, &Operand::Copy(*rhs)); + } + } Rvalue::Aggregate(kind, operands) => { // If we assign `target = Enum::Variant#0(operand)`, // we must make sure that all `target as Variant#i` are `Top`. @@ -138,8 +149,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { variant_target_idx, TrackElem::Field(FieldIdx::from_usize(field_index)), ) { - let result = self.handle_operand(operand, state); - state.insert_idx(field, result, self.map()); + self.assign_operand(state, field, operand); } } } @@ -176,7 +186,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { if let Some(overflow_target) = overflow_target { let overflow = match overflow { FlatSet::Top => FlatSet::Top, - FlatSet::Elem(overflow) => FlatSet::Elem(overflow.into()), + FlatSet::Elem(overflow) => FlatSet::Elem(Scalar::from_bool(overflow)), FlatSet::Bottom => FlatSet::Bottom, }; // We have flooded `target` earlier. @@ -196,9 +206,9 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { && let operand_ty = operand.ty(self.local_decls, self.tcx) && let Some(operand_ty) = operand_ty.builtin_deref(true) && let ty::Array(_, len) = operand_ty.ty.kind() - && let Some(len) = ConstantKind::Ty(*len).eval(self.tcx, self.param_env).try_to_scalar_int() + && let Some(len) = Const::Ty(*len).try_eval_scalar_int(self.tcx, self.param_env) { - state.insert_value_idx(target_len, FlatSet::Elem(len), self.map()); + state.insert_value_idx(target_len, FlatSet::Elem(len.into()), self.map()); } } _ => self.super_assign(target, rvalue, state), @@ -214,9 +224,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { Rvalue::Len(place) => { let place_ty = place.ty(self.local_decls, self.tcx); if let ty::Array(_, len) = place_ty.ty.kind() { - ConstantKind::Ty(*len) - .eval(self.tcx, self.param_env) - .try_to_scalar_int() + Const::Ty(*len) + .try_eval_scalar(self.tcx, self.param_env) .map_or(FlatSet::Top, FlatSet::Elem) } else if let [ProjectionElem::Deref] = place.projection[..] { state.get_len(place.local.into(), self.map()) @@ -225,21 +234,27 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { } } Rvalue::Cast(CastKind::IntToInt | CastKind::IntToFloat, operand, ty) => { + let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + return ValueOrPlace::Value(FlatSet::Top); + }; match self.eval_operand(operand, state) { FlatSet::Elem(op) => self .ecx - .int_to_int_or_float(&op, *ty) - .map_or(FlatSet::Top, |result| self.wrap_immediate(result)), + .int_to_int_or_float(&op, layout) + .map_or(FlatSet::Top, |result| self.wrap_immediate(*result)), FlatSet::Bottom => FlatSet::Bottom, FlatSet::Top => FlatSet::Top, } } Rvalue::Cast(CastKind::FloatToInt | CastKind::FloatToFloat, operand, ty) => { + let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + return ValueOrPlace::Value(FlatSet::Top); + }; match self.eval_operand(operand, state) { FlatSet::Elem(op) => self .ecx - .float_to_float_or_int(&op, *ty) - .map_or(FlatSet::Top, |result| self.wrap_immediate(result)), + .float_to_float_or_int(&op, layout) + .map_or(FlatSet::Top, |result| self.wrap_immediate(*result)), FlatSet::Bottom => FlatSet::Bottom, FlatSet::Top => FlatSet::Top, } @@ -257,9 +272,10 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { val } Rvalue::UnaryOp(op, operand) => match self.eval_operand(operand, state) { - FlatSet::Elem(value) => { - self.ecx.unary_op(*op, &value).map_or(FlatSet::Top, |val| self.wrap_immty(val)) - } + FlatSet::Elem(value) => self + .ecx + .wrapping_unary_op(*op, &value) + .map_or(FlatSet::Top, |val| self.wrap_immediate(*val)), FlatSet::Bottom => FlatSet::Bottom, FlatSet::Top => FlatSet::Top, }, @@ -275,7 +291,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { .bytes(), _ => return ValueOrPlace::Value(FlatSet::Top), }; - ScalarInt::try_from_target_usize(val, self.tcx).map_or(FlatSet::Top, FlatSet::Elem) + FlatSet::Elem(Scalar::from_target_usize(val, &self.tcx)) } Rvalue::Discriminant(place) => state.get_discr(place.as_ref(), self.map()), _ => return self.super_rvalue(rvalue, state), @@ -285,13 +301,12 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { fn handle_constant( &self, - constant: &Constant<'tcx>, + constant: &ConstOperand<'tcx>, _state: &mut State<Self::Value>, ) -> Self::Value { constant - .literal - .eval(self.tcx, self.param_env) - .try_to_scalar_int() + .const_ + .try_eval_scalar(self.tcx, self.param_env) .map_or(FlatSet::Top, FlatSet::Elem) } @@ -330,13 +345,98 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } } + /// The caller must have flooded `place`. + fn assign_operand( + &self, + state: &mut State<FlatSet<Scalar>>, + place: PlaceIndex, + operand: &Operand<'tcx>, + ) { + match operand { + Operand::Copy(rhs) | Operand::Move(rhs) => { + if let Some(rhs) = self.map.find(rhs.as_ref()) { + state.insert_place_idx(place, rhs, &self.map); + } else if rhs.projection.first() == Some(&PlaceElem::Deref) + && let FlatSet::Elem(pointer) = state.get(rhs.local.into(), &self.map) + && let rhs_ty = self.local_decls[rhs.local].ty + && let Ok(rhs_layout) = self.tcx.layout_of(self.param_env.and(rhs_ty)) + { + let op = ImmTy::from_scalar(pointer, rhs_layout).into(); + self.assign_constant(state, place, op, &rhs.projection); + } + } + Operand::Constant(box constant) => { + if let Ok(constant) = self.ecx.eval_mir_constant(&constant.const_, None, None) { + self.assign_constant(state, place, constant, &[]); + } + } + } + } + + /// The caller must have flooded `place`. + /// + /// Perform: `place = operand.projection`. + #[instrument(level = "trace", skip(self, state))] + fn assign_constant( + &self, + state: &mut State<FlatSet<Scalar>>, + place: PlaceIndex, + mut operand: OpTy<'tcx>, + projection: &[PlaceElem<'tcx>], + ) -> Option<!> { + for &(mut proj_elem) in projection { + if let PlaceElem::Index(index) = proj_elem { + if let FlatSet::Elem(index) = state.get(index.into(), &self.map) + && let Ok(offset) = index.to_target_usize(&self.tcx) + && let Some(min_length) = offset.checked_add(1) + { + proj_elem = PlaceElem::ConstantIndex { offset, min_length, from_end: false }; + } else { + return None; + } + } + operand = self.ecx.project(&operand, proj_elem).ok()?; + } + + self.map.for_each_projection_value( + place, + operand, + &mut |elem, op| match elem { + TrackElem::Field(idx) => self.ecx.project_field(op, idx.as_usize()).ok(), + TrackElem::Variant(idx) => self.ecx.project_downcast(op, idx).ok(), + TrackElem::Discriminant => { + let variant = self.ecx.read_discriminant(op).ok()?; + let discr_value = self.ecx.discriminant_for_variant(op.layout, variant).ok()?; + Some(discr_value.into()) + } + TrackElem::DerefLen => { + let op: OpTy<'_> = self.ecx.deref_pointer(op).ok()?.into(); + let len_usize = op.len(&self.ecx).ok()?; + let layout = + self.tcx.layout_of(self.param_env.and(self.tcx.types.usize)).unwrap(); + Some(ImmTy::from_uint(len_usize, layout).into()) + } + }, + &mut |place, op| { + if let Ok(imm) = self.ecx.read_immediate_raw(op) + && let Some(imm) = imm.right() + { + let elem = self.wrap_immediate(*imm); + state.insert_value_idx(place, elem, &self.map); + } + }, + ); + + None + } + fn binary_op( &self, - state: &mut State<FlatSet<ScalarInt>>, + state: &mut State<FlatSet<Scalar>>, op: BinOp, left: &Operand<'tcx>, right: &Operand<'tcx>, - ) -> (FlatSet<ScalarInt>, FlatSet<bool>) { + ) -> (FlatSet<Scalar>, FlatSet<bool>) { let left = self.eval_operand(left, state); let right = self.eval_operand(right, state); @@ -345,8 +445,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { // Both sides are known, do the actual computation. (FlatSet::Elem(left), FlatSet::Elem(right)) => { match self.ecx.overflowing_binary_op(op, &left, &right) { - Ok((Scalar::Int(val), overflow, _)) => { - (FlatSet::Elem(val), FlatSet::Elem(overflow)) + Ok((val, overflow)) => { + (FlatSet::Elem(val.to_scalar()), FlatSet::Elem(overflow)) } _ => (FlatSet::Top, FlatSet::Top), } @@ -359,9 +459,6 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } let arg_scalar = const_arg.to_scalar(); - let Ok(arg_scalar) = arg_scalar.try_to_int() else { - return (FlatSet::Top, FlatSet::Top); - }; let Ok(arg_value) = arg_scalar.to_bits(layout.size) else { return (FlatSet::Top, FlatSet::Top); }; @@ -387,7 +484,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { fn eval_operand( &self, op: &Operand<'tcx>, - state: &mut State<FlatSet<ScalarInt>>, + state: &mut State<FlatSet<Scalar>>, ) -> FlatSet<ImmTy<'tcx>> { let value = match self.handle_operand(op, state) { ValueOrPlace::Value(value) => value, @@ -397,74 +494,83 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { FlatSet::Top => FlatSet::Top, FlatSet::Elem(scalar) => { let ty = op.ty(self.local_decls, self.tcx); - self.tcx - .layout_of(self.param_env.and(ty)) - .map(|layout| FlatSet::Elem(ImmTy::from_scalar(scalar.into(), layout))) - .unwrap_or(FlatSet::Top) + self.tcx.layout_of(self.param_env.and(ty)).map_or(FlatSet::Top, |layout| { + FlatSet::Elem(ImmTy::from_scalar(scalar.into(), layout)) + }) } FlatSet::Bottom => FlatSet::Bottom, } } - fn eval_discriminant(&self, enum_ty: Ty<'tcx>, variant_index: VariantIdx) -> Option<ScalarInt> { + fn eval_discriminant(&self, enum_ty: Ty<'tcx>, variant_index: VariantIdx) -> Option<Scalar> { if !enum_ty.is_enum() { return None; } - let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?; - let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?; - let discr_value = ScalarInt::try_from_uint(discr.val, discr_layout.size)?; - Some(discr_value) + let enum_ty_layout = self.tcx.layout_of(self.param_env.and(enum_ty)).ok()?; + let discr_value = self.ecx.discriminant_for_variant(enum_ty_layout, variant_index).ok()?; + Some(discr_value.to_scalar()) } - fn wrap_immediate(&self, imm: Immediate) -> FlatSet<ScalarInt> { + fn wrap_immediate(&self, imm: Immediate) -> FlatSet<Scalar> { match imm { - Immediate::Scalar(Scalar::Int(scalar)) => FlatSet::Elem(scalar), + Immediate::Scalar(scalar) => FlatSet::Elem(scalar), + Immediate::Uninit => FlatSet::Bottom, _ => FlatSet::Top, } } - - fn wrap_immty(&self, val: ImmTy<'tcx>) -> FlatSet<ScalarInt> { - self.wrap_immediate(*val) - } } -struct CollectAndPatch<'tcx, 'locals> { +pub(crate) struct Patch<'tcx> { tcx: TyCtxt<'tcx>, - local_decls: &'locals LocalDecls<'tcx>, /// For a given MIR location, this stores the values of the operands used by that location. In /// particular, this is before the effect, such that the operands of `_1 = _1 + _2` are /// properly captured. (This may become UB soon, but it is currently emitted even by safe code.) - before_effect: FxHashMap<(Location, Place<'tcx>), ScalarInt>, + pub(crate) before_effect: FxHashMap<(Location, Place<'tcx>), Const<'tcx>>, /// Stores the assigned values for assignments where the Rvalue is constant. - assignments: FxHashMap<Location, ScalarInt>, + pub(crate) assignments: FxHashMap<Location, Const<'tcx>>, } -impl<'tcx, 'locals> CollectAndPatch<'tcx, 'locals> { - fn new(tcx: TyCtxt<'tcx>, local_decls: &'locals LocalDecls<'tcx>) -> Self { - Self { - tcx, - local_decls, - before_effect: FxHashMap::default(), - assignments: FxHashMap::default(), - } +impl<'tcx> Patch<'tcx> { + pub(crate) fn new(tcx: TyCtxt<'tcx>) -> Self { + Self { tcx, before_effect: FxHashMap::default(), assignments: FxHashMap::default() } + } + + fn make_operand(&self, const_: Const<'tcx>) -> Operand<'tcx> { + Operand::Constant(Box::new(ConstOperand { span: DUMMY_SP, user_ty: None, const_ })) + } +} + +struct Collector<'tcx, 'locals> { + patch: Patch<'tcx>, + local_decls: &'locals LocalDecls<'tcx>, +} + +impl<'tcx, 'locals> Collector<'tcx, 'locals> { + pub(crate) fn new(tcx: TyCtxt<'tcx>, local_decls: &'locals LocalDecls<'tcx>) -> Self { + Self { patch: Patch::new(tcx), local_decls } } - fn make_operand(&self, scalar: ScalarInt, ty: Ty<'tcx>) -> Operand<'tcx> { - Operand::Constant(Box::new(Constant { - span: DUMMY_SP, - user_ty: None, - literal: ConstantKind::Val(ConstValue::Scalar(scalar.into()), ty), - })) + fn try_make_constant( + &self, + place: Place<'tcx>, + state: &State<FlatSet<Scalar>>, + map: &Map, + ) -> Option<Const<'tcx>> { + let FlatSet::Elem(Scalar::Int(value)) = state.get(place.as_ref(), &map) else { + return None; + }; + let ty = place.ty(self.local_decls, self.patch.tcx).ty; + Some(Const::Val(ConstValue::Scalar(value.into()), ty)) } } impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>> - for CollectAndPatch<'tcx, '_> + for Collector<'tcx, '_> { - type FlowState = State<FlatSet<ScalarInt>>; + type FlowState = State<FlatSet<Scalar>>; fn visit_statement_before_primary_effect( &mut self, @@ -494,14 +600,8 @@ impl<'mir, 'tcx> // Don't overwrite the assignment if it already uses a constant (to keep the span). } StatementKind::Assign(box (place, _)) => { - match state.get(place.as_ref(), &results.analysis.0.map) { - FlatSet::Top => (), - FlatSet::Elem(value) => { - self.assignments.insert(location, value); - } - FlatSet::Bottom => { - // This assignment is either unreachable, or an uninitialized value is assigned. - } + if let Some(value) = self.try_make_constant(place, state, &results.analysis.0.map) { + self.patch.assignments.insert(location, value); } } _ => (), @@ -520,7 +620,7 @@ impl<'mir, 'tcx> } } -impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { +impl<'tcx> MutVisitor<'tcx> for Patch<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -529,8 +629,7 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { if let Some(value) = self.assignments.get(&location) { match &mut statement.kind { StatementKind::Assign(box (_, rvalue)) => { - let ty = rvalue.ty(self.local_decls, self.tcx); - *rvalue = Rvalue::Use(self.make_operand(*value, ty)); + *rvalue = Rvalue::Use(self.make_operand(*value)); } _ => bug!("found assignment info for non-assign statement"), } @@ -543,8 +642,7 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { match operand { Operand::Copy(place) | Operand::Move(place) => { if let Some(value) = self.before_effect.get(&(location, *place)) { - let ty = place.ty(self.local_decls, self.tcx).ty; - *operand = self.make_operand(*value, ty); + *operand = self.make_operand(*value); } else if !place.projection.is_empty() { self.super_operand(operand, location) } @@ -558,11 +656,11 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { elem: PlaceElem<'tcx>, location: Location, ) -> Option<PlaceElem<'tcx>> { - if let PlaceElem::Index(local) = elem - && let Some(value) = self.before_effect.get(&(location, local.into())) - && let Ok(offset) = value.try_to_target_usize(self.tcx) - && let Some(min_length) = offset.checked_add(1) - { + if let PlaceElem::Index(local) = elem { + let offset = self.before_effect.get(&(location, local.into()))?; + let offset = offset.try_to_scalar()?; + let offset = offset.to_target_usize(&self.tcx).ok()?; + let min_length = offset.checked_add(1)?; Some(PlaceElem::ConstantIndex { offset, min_length, from_end: false }) } else { None @@ -571,30 +669,36 @@ impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx, '_> { } struct OperandCollector<'tcx, 'map, 'locals, 'a> { - state: &'a State<FlatSet<ScalarInt>>, - visitor: &'a mut CollectAndPatch<'tcx, 'locals>, + state: &'a State<FlatSet<Scalar>>, + visitor: &'a mut Collector<'tcx, 'locals>, map: &'map Map, } impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> { + fn visit_projection_elem( + &mut self, + _: PlaceRef<'tcx>, + elem: PlaceElem<'tcx>, + _: PlaceContext, + location: Location, + ) { + if let PlaceElem::Index(local) = elem + && let Some(value) = self.visitor.try_make_constant(local.into(), self.state, self.map) + { + self.visitor.patch.before_effect.insert((location, local.into()), value); + } + } + fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { if let Some(place) = operand.place() { - if let FlatSet::Elem(value) = self.state.get(place.as_ref(), self.map) { - self.visitor.before_effect.insert((location, place), value); + if let Some(value) = self.visitor.try_make_constant(place, self.state, self.map) { + self.visitor.patch.before_effect.insert((location, place), value); } else if !place.projection.is_empty() { // Try to propagate into `Index` projections. self.super_operand(operand, location) } } } - - fn visit_local(&mut self, local: Local, ctxt: PlaceContext, location: Location) { - if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy | NonMutatingUseContext::Move) = ctxt - && let FlatSet::Elem(value) = self.state.get(local.into(), self.map) - { - self.visitor.before_effect.insert((location, local.into()), value); - } - } } struct DummyMachine; @@ -604,8 +708,11 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm type MemoryKind = !; const PANIC_ON_ALLOC_FAIL: bool = true; + #[inline(always)] fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> CheckAlignment { - unimplemented!() + // We do not check for alignment to avoid having to carry an `Align` + // in `ConstValue::ByRef`. + CheckAlignment::No } fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool { @@ -620,6 +727,27 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm unimplemented!() } + fn before_access_global( + _tcx: TyCtxt<'tcx>, + _machine: &Self, + _alloc_id: AllocId, + alloc: ConstAllocation<'tcx>, + _static_def_id: Option<DefId>, + is_write: bool, + ) -> InterpResult<'tcx> { + if is_write { + crate::const_prop::throw_machine_stop_str!("can't write to global"); + } + + // If the static allocation is mutable, then we can't const prop it as its content + // might be different at runtime. + if alloc.inner().mutability.is_mut() { + crate::const_prop::throw_machine_stop_str!("can't access mutable globals in ConstProp"); + } + + Ok(()) + } + fn find_mir_or_eval_fn( _ecx: &mut InterpCx<'mir, 'tcx, Self>, _instance: ty::Instance<'tcx>, @@ -663,8 +791,8 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm _bin_op: BinOp, _left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>, _right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>, - ) -> interpret::InterpResult<'tcx, (Scalar<Self::Provenance>, bool, Ty<'tcx>)> { - throw_unsup!(Unsupported("".into())) + ) -> interpret::InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)> { + crate::const_prop::throw_machine_stop_str!("can't do pointer arithmetic"); } fn expose_ptr( @@ -688,7 +816,8 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm _ecx: &'a InterpCx<'mir, 'tcx, Self>, ) -> &'a [rustc_const_eval::interpret::Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] { - unimplemented!() + // Return an empty stack instead of panicking, as `cur_span` uses it to evaluate constants. + &[] } fn stack_mut<'a>( diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs index 909116a77f5..666293cbc30 100644 --- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs +++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs @@ -150,7 +150,7 @@ fn rvalue_hash<H: Hasher>(hasher: &mut H, rvalue: &Rvalue<'_>) { fn operand_hash<H: Hasher>(hasher: &mut H, operand: &Operand<'_>) { match operand { - Operand::Constant(box Constant { user_ty: _, literal, span: _ }) => literal.hash(hasher), + Operand::Constant(box ConstOperand { user_ty: _, const_, span: _ }) => const_.hash(hasher), x => x.hash(hasher), }; } @@ -179,9 +179,9 @@ fn rvalue_eq<'tcx>(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { fn operand_eq<'tcx>(lhs: &Operand<'tcx>, rhs: &Operand<'tcx>) -> bool { let res = match (lhs, rhs) { ( - Operand::Constant(box Constant { user_ty: _, literal, span: _ }), - Operand::Constant(box Constant { user_ty: _, literal: literal2, span: _ }), - ) => literal == literal2, + Operand::Constant(box ConstOperand { user_ty: _, const_, span: _ }), + Operand::Constant(box ConstOperand { user_ty: _, const_: const2, span: _ }), + ) => const_ == const2, (x, y) => x == y, }; debug!("operand_eq lhs: `{:?}` rhs: `{:?}` result: {:?}", lhs, rhs, res); diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 78d7ffb3698..b62d7da2a4c 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -170,6 +170,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> { self.ctxt.param_env() } + #[instrument(level = "debug", skip(self), ret)] fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle { let ((maybe_live, maybe_dead), multipart) = match mode { DropFlagMode::Shallow => (self.ctxt.init_data.maybe_live_dead(path), false), @@ -402,10 +403,10 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } fn constant_bool(&self, span: Span, val: bool) -> Rvalue<'tcx> { - Rvalue::Use(Operand::Constant(Box::new(Constant { + Rvalue::Use(Operand::Constant(Box::new(ConstOperand { span, user_ty: None, - literal: ConstantKind::from_bool(self.tcx, val), + const_: Const::from_bool(self.tcx, val), }))) } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 35373bcaa41..5879a803946 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -4,7 +4,9 @@ use rustc_errors::{ }; use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_middle::mir::{AssertKind, UnsafetyViolationDetails}; +use rustc_middle::ty::TyCtxt; use rustc_session::lint::{self, Lint}; +use rustc_span::def_id::DefId; use rustc_span::Span; #[derive(LintDiagnostic)] @@ -237,20 +239,38 @@ pub(crate) struct FnItemRef { pub ident: String, } -#[derive(LintDiagnostic)] -#[diag(mir_transform_must_not_suspend)] -pub(crate) struct MustNotSupend<'a> { - #[label] +pub(crate) struct MustNotSupend<'tcx, 'a> { + pub tcx: TyCtxt<'tcx>, pub yield_sp: Span, - #[subdiagnostic] pub reason: Option<MustNotSuspendReason>, - #[help] pub src_sp: Span, pub pre: &'a str, - pub def_path: String, + pub def_id: DefId, pub post: &'a str, } +// Needed for def_path_str +impl<'a> DecorateLint<'a, ()> for MustNotSupend<'_, '_> { + fn decorate_lint<'b>( + self, + diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>, + ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> { + diag.span_label(self.yield_sp, crate::fluent_generated::_subdiag::label); + if let Some(reason) = self.reason { + diag.subdiagnostic(reason); + } + diag.span_help(self.src_sp, crate::fluent_generated::_subdiag::help); + diag.set_arg("pre", self.pre); + diag.set_arg("def_path", self.tcx.def_path_str(self.def_id)); + diag.set_arg("post", self.post); + diag + } + + fn msg(&self) -> rustc_errors::DiagnosticMessage { + crate::fluent_generated::mir_transform_must_not_suspend + } +} + #[derive(Subdiagnostic)] #[note(mir_transform_note)] pub(crate) struct MustNotSuspendReason { diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index eae83448c46..8a807d786a5 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -853,60 +853,7 @@ impl StorageConflictVisitor<'_, '_, '_> { } } -/// Validates the typeck view of the generator against the actual set of types saved between -/// yield points. -fn sanitize_witness<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - witness: Ty<'tcx>, - upvars: &'tcx ty::List<Ty<'tcx>>, - layout: &GeneratorLayout<'tcx>, -) { - let did = body.source.def_id(); - let param_env = tcx.param_env(did); - - let allowed_upvars = tcx.normalize_erasing_regions(param_env, upvars); - let allowed = match witness.kind() { - &ty::GeneratorWitness(interior_tys) => { - tcx.normalize_erasing_late_bound_regions(param_env, interior_tys) - } - _ => { - tcx.sess.delay_span_bug( - body.span, - format!("unexpected generator witness type {:?}", witness.kind()), - ); - return; - } - }; - - let mut mismatches = Vec::new(); - for fty in &layout.field_tys { - if fty.ignore_for_traits { - continue; - } - let decl_ty = tcx.normalize_erasing_regions(param_env, fty.ty); - - // Sanity check that typeck knows about the type of locals which are - // live across a suspension point - if !allowed.contains(&decl_ty) && !allowed_upvars.contains(&decl_ty) { - mismatches.push(decl_ty); - } - } - - if !mismatches.is_empty() { - span_bug!( - body.span, - "Broken MIR: generator contains type {:?} in MIR, \ - but typeck only knows about {} and {:?}", - mismatches, - allowed, - allowed_upvars - ); - } -} - fn compute_layout<'tcx>( - tcx: TyCtxt<'tcx>, liveness: LivenessInfo, body: &Body<'tcx>, ) -> ( @@ -932,27 +879,20 @@ fn compute_layout<'tcx>( let decl = &body.local_decls[local]; debug!(?decl); - let ignore_for_traits = if tcx.sess.opts.unstable_opts.drop_tracking_mir { - // Do not `assert_crate_local` here, as post-borrowck cleanup may have already cleared - // the information. This is alright, since `ignore_for_traits` is only relevant when - // this code runs on pre-cleanup MIR, and `ignore_for_traits = false` is the safer - // default. - match decl.local_info { - // Do not include raw pointers created from accessing `static` items, as those could - // well be re-created by another access to the same static. - ClearCrossCrate::Set(box LocalInfo::StaticRef { is_thread_local, .. }) => { - !is_thread_local - } - // Fake borrows are only read by fake reads, so do not have any reality in - // post-analysis MIR. - ClearCrossCrate::Set(box LocalInfo::FakeBorrow) => true, - _ => false, + // Do not `assert_crate_local` here, as post-borrowck cleanup may have already cleared + // the information. This is alright, since `ignore_for_traits` is only relevant when + // this code runs on pre-cleanup MIR, and `ignore_for_traits = false` is the safer + // default. + let ignore_for_traits = match decl.local_info { + // Do not include raw pointers created from accessing `static` items, as those could + // well be re-created by another access to the same static. + ClearCrossCrate::Set(box LocalInfo::StaticRef { is_thread_local, .. }) => { + !is_thread_local } - } else { - // FIXME(#105084) HIR-based drop tracking does not account for all the temporaries that - // MIR building may introduce. This leads to wrongly ignored types, but this is - // necessary for internal consistency and to avoid ICEs. - decl.internal + // Fake borrows are only read by fake reads, so do not have any reality in + // post-analysis MIR. + ClearCrossCrate::Set(box LocalInfo::FakeBorrow) => true, + _ => false, }; let decl = GeneratorSavedTy { ty: decl.ty, source_info: decl.source_info, ignore_for_traits }; @@ -1189,10 +1129,10 @@ fn insert_panic_block<'tcx>( ) -> BasicBlock { let assert_block = BasicBlock::new(body.basic_blocks.len()); let term = TerminatorKind::Assert { - cond: Operand::Constant(Box::new(Constant { + cond: Operand::Constant(Box::new(ConstOperand { span: body.span, user_ty: None, - literal: ConstantKind::from_bool(tcx, false), + const_: Const::from_bool(tcx, false), })), expected: true, msg: Box::new(message), @@ -1445,8 +1385,6 @@ pub(crate) fn mir_generator_witnesses<'tcx>( tcx: TyCtxt<'tcx>, def_id: LocalDefId, ) -> Option<GeneratorLayout<'tcx>> { - assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir); - let (body, _) = tcx.mir_promoted(def_id); let body = body.borrow(); let body = &*body; @@ -1469,7 +1407,7 @@ pub(crate) fn mir_generator_witnesses<'tcx>( // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto generator struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, generator_layout, _) = compute_layout(tcx, liveness_info, body); + let (_, generator_layout, _) = compute_layout(liveness_info, body); check_suspend_tys(tcx, &generator_layout, &body); @@ -1489,15 +1427,10 @@ impl<'tcx> MirPass<'tcx> for StateTransform { let gen_ty = body.local_decls.raw[1].ty; // Get the discriminant type and args which typeck computed - let (discr_ty, upvars, interior, movable) = match *gen_ty.kind() { + let (discr_ty, movable) = match *gen_ty.kind() { ty::Generator(_, args, movability) => { let args = args.as_generator(); - ( - args.discr_ty(tcx), - args.upvar_tys(), - args.witness(), - movability == hir::Movability::Movable, - ) + (args.discr_ty(tcx), movability == hir::Movability::Movable) } _ => { tcx.sess.delay_span_bug(body.span, format!("unexpected generator type {gen_ty}")); @@ -1574,13 +1507,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto generator struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(tcx, liveness_info, body); - - if tcx.sess.opts.unstable_opts.validate_mir - && !tcx.sess.opts.unstable_opts.drop_tracking_mir - { - sanitize_witness(tcx, body, interior, upvars, &layout); - } + let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id())); @@ -1954,11 +1881,12 @@ fn check_must_not_suspend_def( hir_id, data.source_span, errors::MustNotSupend { + tcx, yield_sp: data.yield_span, reason, src_sp: data.source_span, pre: data.descr_pre, - def_path: tcx.def_path_str(def_id), + def_id, post: data.descr_post, }, ); diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs new file mode 100644 index 00000000000..449bade3322 --- /dev/null +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -0,0 +1,538 @@ +//! Global value numbering. +//! +//! MIR may contain repeated and/or redundant computations. The objective of this pass is to detect +//! such redundancies and re-use the already-computed result when possible. +//! +//! In a first pass, we compute a symbolic representation of values that are assigned to SSA +//! locals. This symbolic representation is defined by the `Value` enum. Each produced instance of +//! `Value` is interned as a `VnIndex`, which allows us to cheaply compute identical values. +//! +//! From those assignments, we construct a mapping `VnIndex -> Vec<(Local, Location)>` of available +//! values, the locals in which they are stored, and a the assignment location. +//! +//! In a second pass, we traverse all (non SSA) assignments `x = rvalue` and operands. For each +//! one, we compute the `VnIndex` of the rvalue. If this `VnIndex` is associated to a constant, we +//! replace the rvalue/operand by that constant. Otherwise, if there is an SSA local `y` +//! associated to this `VnIndex`, and if its definition location strictly dominates the assignment +//! to `x`, we replace the assignment by `x = y`. +//! +//! By opportunity, this pass simplifies some `Rvalue`s based on the accumulated knowledge. +//! +//! # Operational semantic +//! +//! Operationally, this pass attempts to prove bitwise equality between locals. Given this MIR: +//! ```ignore (MIR) +//! _a = some value // has VnIndex i +//! // some MIR +//! _b = some other value // also has VnIndex i +//! ``` +//! +//! We consider it to be replacable by: +//! ```ignore (MIR) +//! _a = some value // has VnIndex i +//! // some MIR +//! _c = some other value // also has VnIndex i +//! assume(_a bitwise equal to _c) // follows from having the same VnIndex +//! _b = _a // follows from the `assume` +//! ``` +//! +//! Which is simplifiable to: +//! ```ignore (MIR) +//! _a = some value // has VnIndex i +//! // some MIR +//! _b = _a +//! ``` +//! +//! # Handling of references +//! +//! We handle references by assigning a different "provenance" index to each Ref/AddressOf rvalue. +//! This ensure that we do not spuriously merge borrows that should not be merged. Meanwhile, we +//! consider all the derefs of an immutable reference to a freeze type to give the same value: +//! ```ignore (MIR) +//! _a = *_b // _b is &Freeze +//! _c = *_b // replaced by _c = _a +//! ``` + +use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::graph::dominators::Dominators; +use rustc_index::bit_set::BitSet; +use rustc_index::IndexVec; +use rustc_macros::newtype_index; +use rustc_middle::mir::visit::*; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_target::abi::{VariantIdx, FIRST_VARIANT}; + +use crate::ssa::SsaLocals; +use crate::MirPass; + +pub struct GVN; + +impl<'tcx> MirPass<'tcx> for GVN { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + + #[instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + debug!(def_id = ?body.source.def_id()); + propagate_ssa(tcx, body); + } +} + +fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let ssa = SsaLocals::new(body); + // Clone dominators as we need them while mutating the body. + let dominators = body.basic_blocks.dominators().clone(); + + let mut state = VnState::new(tcx, param_env, &ssa, &dominators, &body.local_decls); + for arg in body.args_iter() { + if ssa.is_ssa(arg) { + let value = state.new_opaque().unwrap(); + state.assign(arg, value); + } + } + + ssa.for_each_assignment_mut(&mut body.basic_blocks, |local, rvalue, location| { + let value = state.simplify_rvalue(rvalue, location).or_else(|| state.new_opaque()).unwrap(); + // FIXME(#112651) `rvalue` may have a subtype to `local`. We can only mark `local` as + // reusable if we have an exact type match. + if state.local_decls[local].ty == rvalue.ty(state.local_decls, tcx) { + state.assign(local, value); + } + }); + + // Stop creating opaques during replacement as it is useless. + state.next_opaque = None; + + let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec(); + for bb in reverse_postorder { + let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; + state.visit_basic_block_data(bb, data); + } + let any_replacement = state.any_replacement; + + // For each local that is reused (`y` above), we remove its storage statements do avoid any + // difficulty. Those locals are SSA, so should be easy to optimize by LLVM without storage + // statements. + StorageRemover { tcx, reused_locals: state.reused_locals }.visit_body_preserves_cfg(body); + + if any_replacement { + crate::simplify::remove_unused_definitions(body); + } +} + +newtype_index! { + struct VnIndex {} +} + +#[derive(Debug, PartialEq, Eq, Hash)] +enum Value<'tcx> { + // Root values. + /// Used to represent values we know nothing about. + /// The `usize` is a counter incremented by `new_opaque`. + Opaque(usize), + /// Evaluated or unevaluated constant value. + Constant(Const<'tcx>), + /// An aggregate value, either tuple/closure/struct/enum. + /// This does not contain unions, as we cannot reason with the value. + Aggregate(Ty<'tcx>, VariantIdx, Vec<VnIndex>), + /// This corresponds to a `[value; count]` expression. + Repeat(VnIndex, ty::Const<'tcx>), + /// The address of a place. + Address { + place: Place<'tcx>, + /// Give each borrow and pointer a different provenance, so we don't merge them. + provenance: usize, + }, + + // Extractions. + /// This is the *value* obtained by projecting another value. + Projection(VnIndex, ProjectionElem<VnIndex, Ty<'tcx>>), + /// Discriminant of the given value. + Discriminant(VnIndex), + /// Length of an array or slice. + Len(VnIndex), + + // Operations. + NullaryOp(NullOp<'tcx>, Ty<'tcx>), + UnaryOp(UnOp, VnIndex), + BinaryOp(BinOp, VnIndex, VnIndex), + CheckedBinaryOp(BinOp, VnIndex, VnIndex), + Cast { + kind: CastKind, + value: VnIndex, + from: Ty<'tcx>, + to: Ty<'tcx>, + }, +} + +struct VnState<'body, 'tcx> { + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + local_decls: &'body LocalDecls<'tcx>, + /// Value stored in each local. + locals: IndexVec<Local, Option<VnIndex>>, + /// First local to be assigned that value. + rev_locals: FxHashMap<VnIndex, Vec<Local>>, + values: FxIndexSet<Value<'tcx>>, + /// Counter to generate different values. + /// This is an option to stop creating opaques during replacement. + next_opaque: Option<usize>, + ssa: &'body SsaLocals, + dominators: &'body Dominators<BasicBlock>, + reused_locals: BitSet<Local>, + any_replacement: bool, +} + +impl<'body, 'tcx> VnState<'body, 'tcx> { + fn new( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + ssa: &'body SsaLocals, + dominators: &'body Dominators<BasicBlock>, + local_decls: &'body LocalDecls<'tcx>, + ) -> Self { + VnState { + tcx, + param_env, + local_decls, + locals: IndexVec::from_elem(None, local_decls), + rev_locals: FxHashMap::default(), + values: FxIndexSet::default(), + next_opaque: Some(0), + ssa, + dominators, + reused_locals: BitSet::new_empty(local_decls.len()), + any_replacement: false, + } + } + + #[instrument(level = "trace", skip(self), ret)] + fn insert(&mut self, value: Value<'tcx>) -> VnIndex { + let (index, _) = self.values.insert_full(value); + VnIndex::from_usize(index) + } + + /// Create a new `Value` for which we have no information at all, except that it is distinct + /// from all the others. + #[instrument(level = "trace", skip(self), ret)] + fn new_opaque(&mut self) -> Option<VnIndex> { + let next_opaque = self.next_opaque.as_mut()?; + let value = Value::Opaque(*next_opaque); + *next_opaque += 1; + Some(self.insert(value)) + } + + /// Create a new `Value::Address` distinct from all the others. + #[instrument(level = "trace", skip(self), ret)] + fn new_pointer(&mut self, place: Place<'tcx>) -> Option<VnIndex> { + let next_opaque = self.next_opaque.as_mut()?; + let value = Value::Address { place, provenance: *next_opaque }; + *next_opaque += 1; + Some(self.insert(value)) + } + + fn get(&self, index: VnIndex) -> &Value<'tcx> { + self.values.get_index(index.as_usize()).unwrap() + } + + /// Record that `local` is assigned `value`. `local` must be SSA. + #[instrument(level = "trace", skip(self))] + fn assign(&mut self, local: Local, value: VnIndex) { + self.locals[local] = Some(value); + + // Only register the value if its type is `Sized`, as we will emit copies of it. + let is_sized = !self.tcx.features().unsized_locals + || self.local_decls[local].ty.is_sized(self.tcx, self.param_env); + if is_sized { + self.rev_locals.entry(value).or_default().push(local); + } + } + + /// Represent the *value* which would be read from `place`, and point `place` to a preexisting + /// place with the same value (if that already exists). + #[instrument(level = "trace", skip(self), ret)] + fn simplify_place_value( + &mut self, + place: &mut Place<'tcx>, + location: Location, + ) -> Option<VnIndex> { + // Invariant: `place` and `place_ref` point to the same value, even if they point to + // different memory locations. + let mut place_ref = place.as_ref(); + + // Invariant: `value` holds the value up-to the `index`th projection excluded. + let mut value = self.locals[place.local]?; + for (index, proj) in place.projection.iter().enumerate() { + if let Some(local) = self.try_as_local(value, location) { + // Both `local` and `Place { local: place.local, projection: projection[..index] }` + // hold the same value. Therefore, following place holds the value in the original + // `place`. + place_ref = PlaceRef { local, projection: &place.projection[index..] }; + } + + let proj = match proj { + ProjectionElem::Deref => { + let ty = Place::ty_from( + place.local, + &place.projection[..index], + self.local_decls, + self.tcx, + ) + .ty; + if let Some(Mutability::Not) = ty.ref_mutability() + && let Some(pointee_ty) = ty.builtin_deref(true) + && pointee_ty.ty.is_freeze(self.tcx, self.param_env) + { + // An immutable borrow `_x` always points to the same value for the + // lifetime of the borrow, so we can merge all instances of `*_x`. + ProjectionElem::Deref + } else { + return None; + } + } + ProjectionElem::Field(f, ty) => ProjectionElem::Field(f, ty), + ProjectionElem::Index(idx) => { + let idx = self.locals[idx]?; + ProjectionElem::Index(idx) + } + ProjectionElem::ConstantIndex { offset, min_length, from_end } => { + ProjectionElem::ConstantIndex { offset, min_length, from_end } + } + ProjectionElem::Subslice { from, to, from_end } => { + ProjectionElem::Subslice { from, to, from_end } + } + ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index), + ProjectionElem::OpaqueCast(ty) => ProjectionElem::OpaqueCast(ty), + }; + value = self.insert(Value::Projection(value, proj)); + } + + if let Some(local) = self.try_as_local(value, location) + && local != place.local // in case we had no projection to begin with. + { + *place = local.into(); + self.reused_locals.insert(local); + self.any_replacement = true; + } else if place_ref.local != place.local + || place_ref.projection.len() < place.projection.len() + { + // By the invariant on `place_ref`. + *place = place_ref.project_deeper(&[], self.tcx); + self.reused_locals.insert(place_ref.local); + self.any_replacement = true; + } + + Some(value) + } + + #[instrument(level = "trace", skip(self), ret)] + fn simplify_operand( + &mut self, + operand: &mut Operand<'tcx>, + location: Location, + ) -> Option<VnIndex> { + match *operand { + Operand::Constant(ref constant) => Some(self.insert(Value::Constant(constant.const_))), + Operand::Copy(ref mut place) | Operand::Move(ref mut place) => { + let value = self.simplify_place_value(place, location)?; + if let Some(const_) = self.try_as_constant(value) { + *operand = Operand::Constant(Box::new(const_)); + self.any_replacement = true; + } + Some(value) + } + } + } + + #[instrument(level = "trace", skip(self), ret)] + fn simplify_rvalue( + &mut self, + rvalue: &mut Rvalue<'tcx>, + location: Location, + ) -> Option<VnIndex> { + let value = match *rvalue { + // Forward values. + Rvalue::Use(ref mut operand) => return self.simplify_operand(operand, location), + Rvalue::CopyForDeref(place) => { + let mut operand = Operand::Copy(place); + let val = self.simplify_operand(&mut operand, location); + *rvalue = Rvalue::Use(operand); + return val; + } + + // Roots. + Rvalue::Repeat(ref mut op, amount) => { + let op = self.simplify_operand(op, location)?; + Value::Repeat(op, amount) + } + Rvalue::NullaryOp(op, ty) => Value::NullaryOp(op, ty), + Rvalue::Aggregate(box ref kind, ref mut fields) => { + let variant_index = match *kind { + AggregateKind::Array(..) + | AggregateKind::Tuple + | AggregateKind::Closure(..) + | AggregateKind::Generator(..) => FIRST_VARIANT, + AggregateKind::Adt(_, variant_index, _, _, None) => variant_index, + // Do not track unions. + AggregateKind::Adt(_, _, _, _, Some(_)) => return None, + }; + let fields: Option<Vec<_>> = fields + .iter_mut() + .map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque())) + .collect(); + let ty = rvalue.ty(self.local_decls, self.tcx); + Value::Aggregate(ty, variant_index, fields?) + } + Rvalue::Ref(.., place) | Rvalue::AddressOf(_, place) => return self.new_pointer(place), + + // Operations. + Rvalue::Len(ref mut place) => { + let place = self.simplify_place_value(place, location)?; + Value::Len(place) + } + Rvalue::Cast(kind, ref mut value, to) => { + let from = value.ty(self.local_decls, self.tcx); + let value = self.simplify_operand(value, location)?; + Value::Cast { kind, value, from, to } + } + Rvalue::BinaryOp(op, box (ref mut lhs, ref mut rhs)) => { + let lhs = self.simplify_operand(lhs, location); + let rhs = self.simplify_operand(rhs, location); + Value::BinaryOp(op, lhs?, rhs?) + } + Rvalue::CheckedBinaryOp(op, box (ref mut lhs, ref mut rhs)) => { + let lhs = self.simplify_operand(lhs, location); + let rhs = self.simplify_operand(rhs, location); + Value::CheckedBinaryOp(op, lhs?, rhs?) + } + Rvalue::UnaryOp(op, ref mut arg) => { + let arg = self.simplify_operand(arg, location)?; + Value::UnaryOp(op, arg) + } + Rvalue::Discriminant(ref mut place) => { + let place = self.simplify_place_value(place, location)?; + Value::Discriminant(place) + } + + // Unsupported values. + Rvalue::ThreadLocalRef(..) | Rvalue::ShallowInitBox(..) => return None, + }; + debug!(?value); + Some(self.insert(value)) + } +} + +impl<'tcx> VnState<'_, 'tcx> { + /// If `index` is a `Value::Constant`, return the `Constant` to be put in the MIR. + fn try_as_constant(&mut self, index: VnIndex) -> Option<ConstOperand<'tcx>> { + if let Value::Constant(const_) = *self.get(index) { + // Some constants may contain pointers. We need to preserve the provenance of these + // pointers, but not all constants guarantee this: + // - valtrees purposefully do not; + // - ConstValue::Slice does not either. + match const_ { + Const::Ty(c) => match c.kind() { + ty::ConstKind::Value(valtree) => match valtree { + // This is just an integer, keep it. + ty::ValTree::Leaf(_) => {} + ty::ValTree::Branch(_) => return None, + }, + ty::ConstKind::Param(..) + | ty::ConstKind::Unevaluated(..) + | ty::ConstKind::Expr(..) => {} + // Should not appear in runtime MIR. + ty::ConstKind::Infer(..) + | ty::ConstKind::Bound(..) + | ty::ConstKind::Placeholder(..) + | ty::ConstKind::Error(..) => bug!(), + }, + Const::Unevaluated(..) => {} + // If the same slice appears twice in the MIR, we cannot guarantee that we will + // give the same `AllocId` to the data. + Const::Val(ConstValue::Slice { .. }, _) => return None, + Const::Val( + ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. }, + _, + ) => {} + } + Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ }) + } else { + None + } + } + + /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`, + /// return it. + fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option<Local> { + let other = self.rev_locals.get(&index)?; + other + .iter() + .copied() + .find(|&other| self.ssa.assignment_dominates(self.dominators, other, loc)) + } +} + +impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { + self.simplify_operand(operand, location); + } + + fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) { + self.super_statement(stmt, location); + if let StatementKind::Assign(box (_, ref mut rvalue)) = stmt.kind + // Do not try to simplify a constant, it's already in canonical shape. + && !matches!(rvalue, Rvalue::Use(Operand::Constant(_))) + && let Some(value) = self.simplify_rvalue(rvalue, location) + { + if let Some(const_) = self.try_as_constant(value) { + *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); + self.any_replacement = true; + } else if let Some(local) = self.try_as_local(value, location) + && *rvalue != Rvalue::Use(Operand::Move(local.into())) + { + *rvalue = Rvalue::Use(Operand::Copy(local.into())); + self.reused_locals.insert(local); + self.any_replacement = true; + } + } + } +} + +struct StorageRemover<'tcx> { + tcx: TyCtxt<'tcx>, + reused_locals: BitSet<Local>, +} + +impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_operand(&mut self, operand: &mut Operand<'tcx>, _: Location) { + if let Operand::Move(place) = *operand + && let Some(local) = place.as_local() + && self.reused_locals.contains(local) + { + *operand = Operand::Copy(place); + } + } + + fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, loc: Location) { + match stmt.kind { + // When removing storage statements, we need to remove both (#107511). + StatementKind::StorageLive(l) | StatementKind::StorageDead(l) + if self.reused_locals.contains(l) => + { + stmt.make_nop() + } + _ => self.super_statement(stmt, loc), + } + } +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 7d4c4a823a8..ebd61f8ad95 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -193,7 +193,7 @@ impl<'tcx> Inliner<'tcx> { return Err("optimization fuel exhausted"); } - let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( + let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( self.tcx, self.param_env, ty::EarlyBinder::bind(callee_body.clone()), @@ -390,7 +390,12 @@ impl<'tcx> Inliner<'tcx> { // Reachability pass defines which functions are eligible for inlining. Generally inlining // other functions is incorrect because they could reference symbols that aren't exported. - let is_generic = callsite.callee.args.non_erasable_generics().next().is_some(); + let is_generic = callsite + .callee + .args + .non_erasable_generics(self.tcx, callsite.callee.def_id()) + .next() + .is_some(); if !is_generic && !callee_attrs.requests_inline() { return Err("not exported"); } @@ -476,9 +481,10 @@ impl<'tcx> Inliner<'tcx> { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite - .callee - .subst_mir(self.tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); + let ty = callsite.callee.instantiate_mir( + self.tcx, + ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty), + ); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } @@ -645,13 +651,13 @@ impl<'tcx> Inliner<'tcx> { // Copy only unevaluated constants from the callee_body into the caller_body. // Although we are only pushing `ConstKind::Unevaluated` consts to // `required_consts`, here we may not only have `ConstKind::Unevaluated` - // because we are calling `subst_and_normalize_erasing_regions`. + // because we are calling `instantiate_and_normalize_erasing_regions`. caller_body.required_consts.extend( - callee_body.required_consts.iter().copied().filter(|&ct| match ct.literal { - ConstantKind::Ty(_) => { + callee_body.required_consts.iter().copied().filter(|&ct| match ct.const_ { + Const::Ty(_) => { bug!("should never encounter ty::UnevaluatedConst in `required_consts`") } - ConstantKind::Val(..) | ConstantKind::Unevaluated(..) => true, + Const::Val(..) | Const::Unevaluated(..) => true, }), ); } @@ -806,9 +812,10 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { match terminator.kind { TerminatorKind::Drop { ref place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = self - .instance - .subst_mir(tcx, ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty)); + let ty = self.instance.instantiate_mir( + tcx, + ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty), + ); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -819,7 +826,8 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder::bind(&f.literal.ty())); + let fn_ty = + self.instance.instantiate_mir(tcx, ty::EarlyBinder::bind(&f.const_.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 822634129fc..d30e0bad813 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -44,7 +44,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( ) -> bool { trace!(%caller); for &(callee, args) in tcx.mir_inliner_callees(caller.def) { - let Ok(args) = caller.try_subst_mir_and_normalize_erasing_regions( + let Ok(args) = caller.try_instantiate_mir_and_normalize_erasing_regions( tcx, param_env, ty::EarlyBinder::bind(args), diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 8b0a0903d18..a6ef2e11aa8 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -104,7 +104,7 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> { fn try_eval_bool(&self, a: &Operand<'_>) -> Option<bool> { let a = a.constant()?; - if a.literal.ty().is_bool() { a.literal.try_to_bool() } else { None } + if a.const_.ty().is_bool() { a.const_.try_to_bool() } else { None } } /// Transform "&(*a)" ==> "a". @@ -136,8 +136,8 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> { return; } - let literal = ConstantKind::from_const(len, self.tcx); - let constant = Constant { span: source_info.span, literal, user_ty: None }; + let const_ = Const::from_ty_const(len, self.tcx); + let constant = ConstOperand { span: source_info.span, const_, user_ty: None }; *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); } } diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 19108dabdf4..886ff760481 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -54,11 +54,8 @@ impl EnumSizeOpt { let layout = tcx.layout_of(param_env.and(ty)).ok()?; let variants = match &layout.variants { Variants::Single { .. } => return None, - Variants::Multiple { tag_encoding, .. } - if matches!(tag_encoding, TagEncoding::Niche { .. }) => - { - return None; - } + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => return None, + Variants::Multiple { variants, .. } if variants.len() <= 1 => return None, Variants::Multiple { variants, .. } => variants, }; @@ -114,7 +111,7 @@ impl EnumSizeOpt { tcx.data_layout.ptr_sized_integer().align(&tcx.data_layout).abi, Mutability::Not, ); - let alloc = tcx.create_memory_alloc(tcx.mk_const_alloc(alloc)); + let alloc = tcx.reserve_and_set_memory_alloc(tcx.mk_const_alloc(alloc)); Some((*adt_def, num_discrs, *alloc_cache.entry(ty).or_insert(alloc))) } fn optim<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -139,7 +136,6 @@ impl EnumSizeOpt { let (adt_def, num_variants, alloc_id) = self.candidate(tcx, param_env, ty, &mut alloc_cache)?; - let alloc = tcx.global_alloc(alloc_id).unwrap_memory(); let tmp_ty = Ty::new_array(tcx, tcx.types.usize, num_variants as u64); @@ -150,11 +146,11 @@ impl EnumSizeOpt { }; let place = Place::from(size_array_local); - let constant_vals = Constant { + let constant_vals = ConstOperand { span, user_ty: None, - literal: ConstantKind::Val( - interpret::ConstValue::ByRef { alloc, offset: Size::ZERO }, + const_: Const::Val( + ConstValue::Indirect { alloc_id, offset: Size::ZERO }, tmp_ty, ), }; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index bf798adee19..754f2ee8376 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -2,6 +2,7 @@ #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] #![feature(box_patterns)] +#![feature(decl_macro)] #![feature(is_sorted)] #![feature(let_chains)] #![feature(map_try_insert)] @@ -30,9 +31,9 @@ use rustc_hir::intravisit::{self, Visitor}; use rustc_index::IndexVec; use rustc_middle::mir::visit::Visitor as _; use rustc_middle::mir::{ - traversal, AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, - MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, - Statement, StatementKind, TerminatorKind, START_BLOCK, + traversal, AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstOperand, ConstQualifs, + LocalDecl, MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, + SourceInfo, Statement, StatementKind, TerminatorKind, START_BLOCK, }; use rustc_middle::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; @@ -75,6 +76,7 @@ mod errors; mod ffi_unwind_calls; mod function_item_references; mod generator; +mod gvn; pub mod inline; mod instsimplify; mod large_enums; @@ -148,14 +150,14 @@ fn remap_mir_for_const_eval_select<'tcx>( let terminator = bb.terminator.as_mut().expect("invalid terminator"); match terminator.kind { TerminatorKind::Call { - func: Operand::Constant(box Constant { ref literal, .. }), + func: Operand::Constant(box ConstOperand { ref const_, .. }), ref mut args, destination, target, unwind, fn_span, .. - } if let ty::FnDef(def_id, _) = *literal.ty().kind() + } if let ty::FnDef(def_id, _) = *const_.ty().kind() && tcx.item_name(def_id) == sym::const_eval_select && tcx.is_intrinsic(def_id) => { @@ -342,7 +344,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> { let body = match tcx.hir().body_const_context(def) { // consts and statics do not have `optimized_mir`, so we can steal the body instead of // cloning it. - Some(hir::ConstContext::Const | hir::ConstContext::Static(_)) => body.steal(), + Some(hir::ConstContext::Const { .. } | hir::ConstContext::Static(_)) => body.steal(), Some(hir::ConstContext::ConstFn) => body.borrow().clone(), None => bug!("`mir_for_ctfe` called on non-const {def:?}"), }; @@ -357,9 +359,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> { /// mir borrowck *before* doing so in order to ensure that borrowck can be run and doesn't /// end up missing the source MIR due to stealing happening. fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { - if tcx.sess.opts.unstable_opts.drop_tracking_mir - && let DefKind::Generator = tcx.def_kind(def) - { + if let DefKind::Generator = tcx.def_kind(def) { tcx.ensure_with_value().mir_generator_witnesses(def); } let mir_borrowck = tcx.mir_borrowck(def); @@ -480,6 +480,7 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let passes: &[&dyn MirPass<'tcx>] = &[ // These next passes must be executed together &add_call_guards::CriticalCallEdges, + &reveal_all::RevealAll, // has to be done before drop elaboration, since we need to drop opaque types, too. &elaborate_drops::ElaborateDrops, // This will remove extraneous landing pads which are no longer // necessary as well as well as forcing any call in a non-unwinding @@ -526,7 +527,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { body, &[ &check_alignment::CheckAlignment, - &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first &unreachable_prop::UnreachablePropagation, &uninhabited_enum_branching::UninhabitedEnumBranching, @@ -550,6 +550,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // latter pass will leverage the created opportunities. &separate_const_switch::SeparateConstSwitch, &const_prop::ConstProp, + &gvn::GVN, &dataflow_const_prop::DataflowConstProp, // // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. @@ -605,6 +606,11 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { let body = tcx.mir_drops_elaborated_and_const_checked(did).steal(); let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst); debug!("body: {:#?}", body); + + if body.tainted_by_errors.is_some() { + return body; + } + run_optimization_passes(tcx, &mut body); body diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 13277d62bf4..0d2d764c422 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -32,10 +32,10 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { source_info: terminator.source_info, kind: StatementKind::Assign(Box::new(( *destination, - Rvalue::Use(Operand::Constant(Box::new(Constant { + Rvalue::Use(Operand::Constant(Box::new(ConstOperand { span: terminator.source_info.span, user_ty: None, - literal: ConstantKind::zero_sized(tcx.types.unit), + const_: Const::zero_sized(tcx.types.unit), }))), ))), }); diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index bc29fb8ded1..3dc627b6146 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -98,10 +98,10 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { StatementKind::Assign(box (lhs_f, Rvalue::Use(Operand::Constant(f_c)))), StatementKind::Assign(box (lhs_s, Rvalue::Use(Operand::Constant(s_c)))), ) if lhs_f == lhs_s - && f_c.literal.ty().is_bool() - && s_c.literal.ty().is_bool() - && f_c.literal.try_eval_bool(tcx, param_env).is_some() - && s_c.literal.try_eval_bool(tcx, param_env).is_some() => {} + && f_c.const_.ty().is_bool() + && s_c.const_.ty().is_bool() + && f_c.const_.try_eval_bool(tcx, param_env).is_some() + && s_c.const_.try_eval_bool(tcx, param_env).is_some() => {} // Otherwise we cannot optimize. Try another block. _ => continue 'outer, @@ -128,8 +128,8 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { StatementKind::Assign(box (_, Rvalue::Use(Operand::Constant(s_c)))), ) => { // From earlier loop we know that we are dealing with bool constants only: - let f_b = f_c.literal.try_eval_bool(tcx, param_env).unwrap(); - let s_b = s_c.literal.try_eval_bool(tcx, param_env).unwrap(); + let f_b = f_c.const_.try_eval_bool(tcx, param_env).unwrap(); + let s_b = s_c.const_.try_eval_bool(tcx, param_env).unwrap(); if f_b == s_b { // Same value in both blocks. Use statement as is. (*f).clone() diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index 6c3b7c58fab..d1a4b26a046 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -90,10 +90,10 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { && let [PlaceElem::Deref] = &place.projection[..] && let Some(len) = self.slice_lengths[place.local] { - *rvalue = Rvalue::Use(Operand::Constant(Box::new(Constant { + *rvalue = Rvalue::Use(Operand::Constant(Box::new(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, - literal: ConstantKind::from_const(len, self.tcx), + const_: Const::from_ty_const(len, self.tcx), }))); } self.super_rvalue(rvalue, loc); diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 057f5fe8293..5abb2f3d041 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -94,6 +94,8 @@ fn run_passes_inner<'tcx>( let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; trace!(?overridden_passes); + let prof_arg = tcx.sess.prof.enabled().then(|| format!("{:?}", body.source.def_id())); + if !body.should_skip() { for pass in passes { let name = pass.name(); @@ -121,7 +123,14 @@ fn run_passes_inner<'tcx>( validate_body(tcx, body, format!("before pass {name}")); } - tcx.sess.time(name, || pass.run_pass(tcx, body)); + if let Some(prof_arg) = &prof_arg { + tcx.sess + .prof + .generic_activity_with_arg(pass.profiler_name(), &**prof_arg) + .run(|| pass.run_pass(tcx, body)); + } else { + pass.run_pass(tcx, body); + } if dump_enabled { dump_mir_for_pass(tcx, body, &name, true); diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index 49a940b5779..67941cf4395 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -108,7 +108,7 @@ enum Value<'tcx> { } /// For each local, save the place corresponding to `*local`. -#[instrument(level = "trace", skip(tcx, body))] +#[instrument(level = "trace", skip(tcx, body, ssa))] fn compute_replacement<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index c13bafa9fbb..a34d4b02764 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -1,7 +1,6 @@ //! Removes operations on ZST places, and convert ZST operands to constants. use crate::MirPass; -use rustc_middle::mir::interpret::ConstValue; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; @@ -63,12 +62,12 @@ impl<'tcx> Replacer<'_, 'tcx> { layout.is_zst() } - fn make_zst(&self, ty: Ty<'tcx>) -> Constant<'tcx> { + fn make_zst(&self, ty: Ty<'tcx>) -> ConstOperand<'tcx> { debug_assert!(self.known_to_be_zst(ty)); - Constant { + ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, - literal: ConstantKind::Val(ConstValue::ZeroSized, ty), + const_: Const::Val(ConstValue::ZeroSized, ty), } } } diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 243cb463560..abde6a47e83 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -1,27 +1,27 @@ use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{Constant, ConstantKind, Location}; +use rustc_middle::mir::{Const, ConstOperand, Location}; use rustc_middle::ty::ConstKind; pub struct RequiredConstsVisitor<'a, 'tcx> { - required_consts: &'a mut Vec<Constant<'tcx>>, + required_consts: &'a mut Vec<ConstOperand<'tcx>>, } impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> { - pub fn new(required_consts: &'a mut Vec<Constant<'tcx>>) -> Self { + pub fn new(required_consts: &'a mut Vec<ConstOperand<'tcx>>) -> Self { RequiredConstsVisitor { required_consts } } } impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { - fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) { - let literal = constant.literal; - match literal { - ConstantKind::Ty(c) => match c.kind() { + fn visit_constant(&mut self, constant: &ConstOperand<'tcx>, _: Location) { + let const_ = constant.const_; + match const_ { + Const::Ty(c) => match c.kind() { ConstKind::Param(_) | ConstKind::Error(_) | ConstKind::Value(_) => {} _ => bug!("only ConstKind::Param/Value should be encountered here, got {:#?}", c), }, - ConstantKind::Unevaluated(..) => self.required_consts.push(*constant), - ConstantKind::Val(..) => {} + Const::Unevaluated(..) => self.required_consts.push(*constant), + Const::Val(..) => {} } } } diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 23442f8b97b..55f1eac6f84 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -8,16 +8,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RevealAll; impl<'tcx> MirPass<'tcx> for RevealAll { - fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) - } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // Do not apply this transformation to generators. - if body.generator.is_some() { - return; - } - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); } @@ -35,12 +26,35 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { } #[inline] - fn visit_constant(&mut self, constant: &mut Constant<'tcx>, _: Location) { + fn visit_place( + &mut self, + place: &mut Place<'tcx>, + _context: PlaceContext, + _location: Location, + ) { + // Performance optimization: don't reintern if there is no `OpaqueCast` to remove. + if place.projection.iter().all(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) { + return; + } + // `OpaqueCast` projections are only needed if there are opaque types on which projections are performed. + // After the `RevealAll` pass, all opaque types are replaced with their hidden types, so we don't need these + // projections anymore. + place.projection = self.tcx.mk_place_elems( + &place + .projection + .into_iter() + .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) + .collect::<Vec<_>>(), + ); + } + + #[inline] + fn visit_constant(&mut self, constant: &mut ConstOperand<'tcx>, _: Location) { // We have to use `try_normalize_erasing_regions` here, since it's // possible that we visit impossible-to-satisfy where clauses here, // see #91745 - if let Ok(c) = self.tcx.try_normalize_erasing_regions(self.param_env, constant.literal) { - constant.literal = c; + if let Ok(c) = self.tcx.try_normalize_erasing_regions(self.param_env, constant.const_) { + constant.const_ = c; } } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index e1000d96932..e9895d97dfe 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -497,10 +497,10 @@ impl<'tcx> CloneShimBuilder<'tcx> { // `func == Clone::clone(&ty) -> ty` let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]); - let func = Operand::Constant(Box::new(Constant { + let func = Operand::Constant(Box::new(ConstOperand { span: self.span, user_ty: None, - literal: ConstantKind::zero_sized(func_ty), + const_: Const::zero_sized(func_ty), })); let ref_loc = self.make_place( @@ -764,10 +764,10 @@ fn build_call_shim<'tcx>( CallKind::Direct(def_id) => { let ty = tcx.type_of(def_id).instantiate_identity(); ( - Operand::Constant(Box::new(Constant { + Operand::Constant(Box::new(ConstOperand { span, user_ty: None, - literal: ConstantKind::zero_sized(ty), + const_: Const::zero_sized(ty), })), rcvr.into_iter().collect::<Vec<_>>(), ) diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index b7a51cfd619..2795cf15702 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -29,6 +29,7 @@ use crate::MirPass; use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; +use rustc_index::bit_set::BitSet; use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_middle::mir::coverage::*; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; @@ -345,24 +346,22 @@ pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let basic_blocks = body.basic_blocks.as_mut(); let source_scopes = &body.source_scopes; - let mut replacements: Vec<_> = (0..num_blocks).map(BasicBlock::new).collect(); - let mut used_blocks = 0; - for alive_index in reachable.iter() { - let alive_index = alive_index.index(); - replacements[alive_index] = BasicBlock::new(used_blocks); - if alive_index != used_blocks { - // Swap the next alive block data with the current available slot. Since - // alive_index is non-decreasing this is a valid operation. - basic_blocks.raw.swap(alive_index, used_blocks); - } - used_blocks += 1; - } - if tcx.sess.instrument_coverage() { - save_unreachable_coverage(basic_blocks, source_scopes, used_blocks); + save_unreachable_coverage(basic_blocks, source_scopes, &reachable); } - basic_blocks.raw.truncate(used_blocks); + let mut replacements: Vec<_> = (0..num_blocks).map(BasicBlock::new).collect(); + let mut orig_index = 0; + let mut used_index = 0; + basic_blocks.raw.retain(|_| { + let keep = reachable.contains(BasicBlock::new(orig_index)); + if keep { + replacements[orig_index] = BasicBlock::new(used_index); + used_index += 1; + } + orig_index += 1; + keep + }); for block in basic_blocks { for target in block.terminator_mut().successors_mut() { @@ -404,11 +403,12 @@ pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { fn save_unreachable_coverage( basic_blocks: &mut IndexSlice<BasicBlock, BasicBlockData<'_>>, source_scopes: &IndexSlice<SourceScope, SourceScopeData<'_>>, - first_dead_block: usize, + reachable: &BitSet<BasicBlock>, ) { // Identify instances that still have some live coverage counters left. let mut live = FxHashSet::default(); - for basic_block in &basic_blocks.raw[0..first_dead_block] { + for bb in reachable.iter() { + let basic_block = &basic_blocks[bb]; for statement in &basic_block.statements { let StatementKind::Coverage(coverage) = &statement.kind else { continue }; let CoverageKind::Counter { .. } = coverage.kind else { continue }; @@ -417,7 +417,8 @@ fn save_unreachable_coverage( } } - for block in &mut basic_blocks.raw[..first_dead_block] { + for bb in reachable.iter() { + let block = &mut basic_blocks[bb]; for statement in &mut block.statements { let StatementKind::Coverage(_) = &statement.kind else { continue }; let instance = statement.source_info.scope.inlined_instance(source_scopes); @@ -433,7 +434,11 @@ fn save_unreachable_coverage( // Retain coverage for instances that still have some live counters left. let mut retained_coverage = Vec::new(); - for dead_block in &basic_blocks.raw[first_dead_block..] { + for dead_block in basic_blocks.indices() { + if reachable.contains(dead_block) { + continue; + } + let dead_block = &basic_blocks[dead_block]; for statement in &dead_block.statements { let StatementKind::Coverage(coverage) = &statement.kind else { continue }; let Some(code_region) = &coverage.code_region else { continue }; diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index 1ff48816986..b508cd1c9cc 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -23,7 +23,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { TerminatorKind::SwitchInt { discr: Operand::Constant(ref c), ref targets, .. } => { - let constant = c.literal.try_eval_bits(tcx, param_env, c.ty()); + let constant = c.const_.try_eval_bits(tcx, param_env); if let Some(constant) = constant { let target = targets.target_for_value(constant); TerminatorKind::Goto { target } @@ -33,7 +33,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { } TerminatorKind::Assert { target, cond: Operand::Constant(ref c), expected, .. - } => match c.literal.try_eval_bool(tcx, param_env) { + } => match c.const_.try_eval_bool(tcx, param_env) { Some(v) if v == expected => TerminatorKind::Goto { target }, _ => continue, }, diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index 113ca2fc5ad..1a8cfc41178 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -206,12 +206,12 @@ fn find_branch_value_info<'tcx>( match (left, right) { (Constant(branch_value), Copy(to_switch_on) | Move(to_switch_on)) | (Copy(to_switch_on) | Move(to_switch_on), Constant(branch_value)) => { - let branch_value_ty = branch_value.literal.ty(); + let branch_value_ty = branch_value.const_.ty(); // we only want to apply this optimization if we are matching on integrals (and chars), as it is not possible to switch on floats if !branch_value_ty.is_integral() && !branch_value_ty.is_char() { return None; }; - let branch_value_scalar = branch_value.literal.try_to_scalar()?; + let branch_value_scalar = branch_value.const_.try_to_scalar()?; Some((branch_value_scalar, branch_value_ty, *to_switch_on)) } _ => None, diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 04bc461c815..af9514ed6bb 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -13,7 +13,6 @@ use rustc_middle::middle::resolve_bound_vars::Set1; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -#[derive(Debug)] pub struct SsaLocals { /// Assignments to each local. This defines whether the local is SSA. assignments: IndexVec<Local, Set1<LocationExtended>>, @@ -79,14 +78,10 @@ impl SsaLocals { visitor.assignments[local] = Set1::One(LocationExtended::Arg); } - if body.basic_blocks.len() > 2 { - for (bb, data) in traversal::reverse_postorder(body) { - visitor.visit_basic_block_data(bb, data); - } - } else { - for (bb, data) in body.basic_blocks.iter_enumerated() { - visitor.visit_basic_block_data(bb, data); - } + // For SSA assignments, a RPO visit will see the assignment before it sees any use. + // We only visit reachable nodes: computing `dominates` on an unreachable node ICEs. + for (bb, data) in traversal::reverse_postorder(body) { + visitor.visit_basic_block_data(bb, data); } for var_debug_info in &body.var_debug_info { @@ -129,6 +124,25 @@ impl SsaLocals { self.direct_uses[local] } + pub fn assignment_dominates( + &self, + dominators: &Dominators<BasicBlock>, + local: Local, + location: Location, + ) -> bool { + match self.assignments[local] { + Set1::One(LocationExtended::Arg) => true, + Set1::One(LocationExtended::Plain(ass)) => { + if ass.block == location.block { + ass.statement_index < location.statement_index + } else { + dominators.dominates(ass.block, location.block) + } + } + _ => false, + } + } + pub fn assignments<'a, 'tcx>( &'a self, body: &'a Body<'tcx>, @@ -146,6 +160,24 @@ impl SsaLocals { }) } + pub fn for_each_assignment_mut<'tcx>( + &self, + basic_blocks: &mut BasicBlocks<'tcx>, + mut f: impl FnMut(Local, &mut Rvalue<'tcx>, Location), + ) { + for &local in &self.assignment_order { + if let Set1::One(LocationExtended::Plain(loc)) = self.assignments[local] { + // `loc` must point to a direct assignment to `local`. + let bbs = basic_blocks.as_mut_preserves_cfg(); + let bb = &mut bbs[loc.block]; + let stmt = &mut bb.statements[loc.statement_index]; + let StatementKind::Assign(box (target, ref mut rvalue)) = stmt.kind else { bug!() }; + assert_eq!(target.as_local(), Some(local)); + f(local, rvalue, loc) + } + } + } + /// Compute the equivalence classes for locals, based on copy statements. /// /// The returned vector maps each local to the one it copies. In the following case: diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index bd1724bf842..0b9311a20ef 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -13,7 +13,11 @@ pub struct UnreachablePropagation; impl MirPass<'_> for UnreachablePropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // Enable only under -Zmir-opt-level=2 as this can make programs less debuggable. - sess.mir_opt_level() >= 2 + + // FIXME(#116171) Coverage gets confused by MIR passes that can remove all + // coverage statements from an instrumented function. This pass can be + // re-enabled when coverage codegen is robust against that happening. + sess.mir_opt_level() >= 2 && !sess.instrument_coverage() } fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |