diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/check_const_item_mutation.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/check_unsafety.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop.rs | 269 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop_lint.rs | 203 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/graph.rs | 40 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/mod.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/deref_separator.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/dest_prop.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/generator.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/large_enums.rs | 45 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/lib.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/nrvo.rs | 2 |
12 files changed, 269 insertions, 335 deletions
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 536745d2cfe..3d32c586554 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -24,7 +24,7 @@ struct ConstMutationChecker<'a, 'tcx> { impl<'tcx> ConstMutationChecker<'_, 'tcx> { fn is_const_item(&self, local: Local) -> Option<DefId> { - if let Some(box LocalInfo::ConstRef { def_id }) = self.body.local_decls[local].local_info { + if let LocalInfo::ConstRef { def_id } = *self.body.local_decls[local].local_info() { Some(def_id) } else { None diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index a8ec568eb0d..c4d058e8ecb 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -182,7 +182,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { // If the projection root is an artificial local that we introduced when // desugaring `static`, give a more specific error message // (avoid the general "raw pointer" clause below, that would only be confusing). - if let Some(box LocalInfo::StaticRef { def_id, .. }) = decl.local_info { + if let LocalInfo::StaticRef { def_id, .. } = *decl.local_info() { if self.tcx.is_mutable_static(def_id) { self.require_unsafe( UnsafetyViolationKind::General, diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 9bc4b26db92..de7b8c63fc8 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -1,12 +1,9 @@ //! Propagates constants for early reporting of statically known //! assertion failures -use std::cell::Cell; - use either::Right; use rustc_const_eval::const_eval::CheckAlignment; -use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; @@ -17,7 +14,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::InternalSubsts; use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt}; -use rustc_span::{def_id::DefId, Span}; +use rustc_span::{def_id::DefId, Span, DUMMY_SP}; use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout}; use rustc_target::spec::abi::Abi as CallAbi; use rustc_trait_selection::traits; @@ -25,8 +22,8 @@ use rustc_trait_selection::traits; use crate::MirPass; use rustc_const_eval::interpret::{ self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame, - ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, PlaceTy, - Pointer, Scalar, StackPopCleanup, StackPopUnwind, + ImmTy, Immediate, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer, + Scalar, StackPopCleanup, StackPopUnwind, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -154,24 +151,12 @@ impl<'tcx> MirPass<'tcx> for ConstProp { pub struct ConstPropMachine<'mir, 'tcx> { /// The virtual call stack. stack: Vec<Frame<'mir, 'tcx>>, - /// `OnlyInsideOwnBlock` locals that were written in the current block get erased at the end. - pub written_only_inside_own_block_locals: FxHashSet<Local>, - /// Locals that need to be cleared after every block terminates. - pub only_propagate_inside_block_locals: BitSet<Local>, pub can_const_prop: IndexVec<Local, ConstPropMode>, } impl ConstPropMachine<'_, '_> { - pub fn new( - only_propagate_inside_block_locals: BitSet<Local>, - can_const_prop: IndexVec<Local, ConstPropMode>, - ) -> Self { - Self { - stack: Vec::new(), - written_only_inside_own_block_locals: Default::default(), - only_propagate_inside_block_locals, - can_const_prop, - } + pub fn new(can_const_prop: IndexVec<Local, ConstPropMode>) -> Self { + Self { stack: Vec::new(), can_const_prop } } } @@ -257,16 +242,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> frame: usize, local: Local, ) -> InterpResult<'tcx, &'a mut interpret::Operand<Self::Provenance>> { - if ecx.machine.can_const_prop[local] == ConstPropMode::NoPropagation { - throw_machine_stop_str!("tried to write to a local that is marked as not propagatable") - } - if frame == 0 && ecx.machine.only_propagate_inside_block_locals.contains(local) { - trace!( - "mutating local {:?} which is restricted to its block. \ - Will remove it from const-prop after block is finished.", - local - ); - ecx.machine.written_only_inside_own_block_locals.insert(local); + assert_eq!(frame, 0); + match ecx.machine.can_const_prop[local] { + ConstPropMode::NoPropagation => { + throw_machine_stop_str!( + "tried to write to a local that is marked as not propagatable" + ) + } + ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => {} } ecx.machine.stack[frame].locals[local].access_mut() } @@ -328,9 +311,6 @@ struct ConstPropagator<'mir, 'tcx> { tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, local_decls: &'mir IndexVec<Local, LocalDecl<'tcx>>, - // Because we have `MutVisitor` we can't obtain the `SourceInfo` from a `Location`. So we store - // the last known `SourceInfo` here and just keep revisiting it. - source_info: Option<SourceInfo>, } impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { @@ -374,17 +354,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let param_env = tcx.param_env_reveal_all_normalized(def_id); let can_const_prop = CanConstProp::check(tcx, param_env, body); - let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len()); - for (l, mode) in can_const_prop.iter_enumerated() { - if *mode == ConstPropMode::OnlyInsideOwnBlock { - only_propagate_inside_block_locals.insert(l); - } - } let mut ecx = InterpCx::new( tcx, tcx.def_span(def_id), param_env, - ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), + ConstPropMachine::new(can_const_prop), ); let ret_layout = ecx @@ -411,13 +385,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { ) .expect("failed to push initial stack frame"); - ConstPropagator { - ecx, - tcx, - param_env, - local_decls: &dummy_body.local_decls, - source_info: None, - } + ConstPropagator { ecx, tcx, param_env, local_decls: &dummy_body.local_decls } } fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { @@ -446,10 +414,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local] = LocalState { - value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)), - layout: Cell::new(None), - }; + ecx.frame_mut().locals[local].value = + LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); } /// Returns the value, if any, of evaluating `c`. @@ -492,11 +458,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { scalar, )) = *value { - *operand = self.operand_from_scalar( - scalar, - value.layout.ty, - self.source_info.unwrap().span, - ); + *operand = self.operand_from_scalar(scalar, value.layout.ty); } } } @@ -504,7 +466,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - fn const_prop(&mut self, rvalue: &Rvalue<'tcx>, place: Place<'tcx>) -> Option<()> { + fn check_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Option<()> { // Perform any special handling for specific Rvalue types. // Generally, checks here fall into one of two categories: // 1. Additional checking to provide useful lints to the user @@ -561,7 +523,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return None; } - self.eval_rvalue_with_identities(rvalue, place) + Some(()) } // Attempt to use algebraic identities to eliminate constant expressions @@ -621,20 +583,24 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } /// Creates a new `Operand::Constant` from a `Scalar` value - fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>, span: Span) -> Operand<'tcx> { + fn operand_from_scalar(&self, scalar: Scalar, ty: Ty<'tcx>) -> Operand<'tcx> { Operand::Constant(Box::new(Constant { - span, + span: DUMMY_SP, user_ty: None, literal: ConstantKind::from_scalar(self.tcx, scalar, ty), })) } - fn replace_with_const( - &mut self, - rval: &mut Rvalue<'tcx>, - value: &OpTy<'tcx>, - source_info: SourceInfo, - ) { + fn replace_with_const(&mut self, place: Place<'tcx>, rval: &mut Rvalue<'tcx>) { + // This will return None if the above `const_prop` invocation only "wrote" a + // type whose creation requires no write. E.g. a generator whose initial state + // consists solely of uninitialized memory (so it doesn't capture any locals). + let Some(ref value) = self.get_const(place) else { return }; + if !self.should_const_prop(value) { + return; + } + trace!("replacing {:?}={:?} with {:?}", place, rval, value); + if let Rvalue::Use(Operand::Constant(c)) = rval { match c.literal { ConstantKind::Ty(c) if matches!(c.kind(), ConstKind::Unevaluated(..)) => {} @@ -664,11 +630,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if let Some(Right(imm)) = imm { match *imm { interpret::Immediate::Scalar(scalar) => { - *rval = Rvalue::Use(self.operand_from_scalar( - scalar, - value.layout.ty, - source_info.span, - )); + *rval = Rvalue::Use(self.operand_from_scalar(scalar, value.layout.ty)); } Immediate::ScalarPair(..) => { // Found a value represented as a pair. For now only do const-prop if the type @@ -701,7 +663,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let const_val = ConstValue::ByRef { alloc, offset: Size::ZERO }; let literal = ConstantKind::Val(const_val, ty); *rval = Rvalue::Use(Operand::Constant(Box::new(Constant { - span: source_info.span, + span: DUMMY_SP, user_ty: None, literal, }))); @@ -730,6 +692,19 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { _ => false, } } + + fn ensure_not_propagated(&mut self, local: Local) { + if cfg!(debug_assertions) { + assert!( + self.get_const(local.into()).is_none() + || self + .layout_of(self.local_decls[local].ty) + .map_or(true, |layout| layout.is_zst()), + "failed to remove values for `{local:?}`, value={:?}", + self.get_const(local.into()), + ) + } + } } /// The mode that `ConstProp` is allowed to run in for a given `Local`. @@ -739,8 +714,6 @@ pub enum ConstPropMode { FullConstProp, /// The `Local` can only be propagated into and from its own block. OnlyInsideOwnBlock, - /// The `Local` can be propagated into but reads cannot be propagated. - OnlyPropagateInto, /// The `Local` cannot be part of propagation at all. Any statement /// referencing it either for reading or writing will not get propagated. NoPropagation, @@ -750,8 +723,6 @@ pub struct CanConstProp { can_const_prop: IndexVec<Local, ConstPropMode>, // False at the beginning. Once set, no more assignments are allowed to that local. found_assignment: BitSet<Local>, - // Cache of locals' information - local_kinds: IndexVec<Local, LocalKind>, } impl CanConstProp { @@ -764,10 +735,6 @@ impl CanConstProp { let mut cpv = CanConstProp { can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls), found_assignment: BitSet::new_empty(body.local_decls.len()), - local_kinds: IndexVec::from_fn_n( - |local| body.local_kind(local), - body.local_decls.len(), - ), }; for (local, val) in cpv.can_const_prop.iter_enumerated_mut() { let ty = body.local_decls[local].ty; @@ -780,24 +747,10 @@ impl CanConstProp { continue; } } - // Cannot use args at all - // Cannot use locals because if x < y { y - x } else { x - y } would - // lint for x != y - // FIXME(oli-obk): lint variables until they are used in a condition - // FIXME(oli-obk): lint if return value is constant - if cpv.local_kinds[local] == LocalKind::Arg { - *val = ConstPropMode::OnlyPropagateInto; - trace!( - "local {:?} can't be const propagated because it's a function argument", - local - ); - } else if cpv.local_kinds[local] == LocalKind::Var { - *val = ConstPropMode::OnlyInsideOwnBlock; - trace!( - "local {:?} will only be propagated inside its block, because it's a user variable", - local - ); - } + } + // Consider that arguments are assigned on entry. + for arg in body.args_iter() { + cpv.found_assignment.insert(arg); } cpv.visit_body(&body); cpv.can_const_prop @@ -827,7 +780,6 @@ impl Visitor<'_> for CanConstProp { // states as applicable. ConstPropMode::OnlyInsideOwnBlock => {} ConstPropMode::NoPropagation => {} - ConstPropMode::OnlyPropagateInto => {} other @ ConstPropMode::FullConstProp => { trace!( "local {:?} can't be propagated because of multiple assignments. Previous state: {:?}", @@ -892,42 +844,23 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { self.eval_constant(constant); } - fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { - trace!("visit_statement: {:?}", statement); - let source_info = statement.source_info; - self.source_info = Some(source_info); - match statement.kind { - StatementKind::Assign(box (place, ref mut rval)) => { - let can_const_prop = self.ecx.machine.can_const_prop[place.local]; - if let Some(()) = self.const_prop(rval, place) { - // This will return None if the above `const_prop` invocation only "wrote" a - // type whose creation requires no write. E.g. a generator whose initial state - // consists solely of uninitialized memory (so it doesn't capture any locals). - if let Some(ref value) = self.get_const(place) && self.should_const_prop(value) { - trace!("replacing {:?} with {:?}", rval, value); - self.replace_with_const(rval, value, source_info); - if can_const_prop == ConstPropMode::FullConstProp - || can_const_prop == ConstPropMode::OnlyInsideOwnBlock - { - trace!("propagated into {:?}", place); - } - } - match can_const_prop { - ConstPropMode::OnlyInsideOwnBlock => { - trace!( - "found local restricted to its block. \ - Will remove it from const-prop after block is finished. Local: {:?}", - place.local - ); - } - ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { - trace!("can't propagate into {:?}", place); - if place.local != RETURN_PLACE { - Self::remove_const(&mut self.ecx, place.local); - } - } - ConstPropMode::FullConstProp => {} - } + fn visit_assign( + &mut self, + place: &mut Place<'tcx>, + rvalue: &mut Rvalue<'tcx>, + location: Location, + ) { + self.super_assign(place, rvalue, location); + + let Some(()) = self.check_rvalue(rvalue) else { return }; + + match self.ecx.machine.can_const_prop[place.local] { + // Do nothing if the place is indirect. + _ if place.is_indirect() => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), + ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => { + if let Some(()) = self.eval_rvalue_with_identities(rvalue, *place) { + self.replace_with_const(*place, rvalue); } else { // Const prop failed, so erase the destination, ensuring that whatever happens // from here on, does not know about the previous value. @@ -947,8 +880,22 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { Self::remove_const(&mut self.ecx, place.local); } } + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + trace!("visit_statement: {:?}", statement); + + // We want to evaluate operands before any change to the assigned-to value, + // so we recurse first. + self.super_statement(statement, location); + + match statement.kind { StatementKind::SetDiscriminant { ref place, .. } => { match self.ecx.machine.can_const_prop[place.local] { + // Do nothing if the place is indirect. + _ if place.is_indirect() => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { if self.ecx.statement(statement).is_ok() { trace!("propped discriminant into {:?}", place); @@ -956,28 +903,22 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { Self::remove_const(&mut self.ecx, place.local); } } - ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { - Self::remove_const(&mut self.ecx, place.local); - } } } - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + StatementKind::StorageLive(local) => { let frame = self.ecx.frame_mut(); - frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind { - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)) - } else { - LocalValue::Dead - }; + frame.locals[local].value = + LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + } + StatementKind::StorageDead(local) => { + let frame = self.ecx.frame_mut(); + frame.locals[local].value = LocalValue::Dead; } _ => {} } - - self.super_statement(statement, location); } fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { - let source_info = terminator.source_info; - self.source_info = Some(source_info); self.super_terminator(terminator, location); match &mut terminator.kind { @@ -987,11 +928,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { && self.should_const_prop(value) { trace!("assertion on {:?} should be {:?}", value, expected); - *cond = self.operand_from_scalar( - value_const, - self.tcx.types.bool, - source_info.span, - ); + *cond = self.operand_from_scalar(value_const, self.tcx.types.bool); } } TerminatorKind::SwitchInt { ref mut discr, .. } => { @@ -1026,23 +963,17 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); - for &local in locals.iter() { - Self::remove_const(&mut self.ecx, local); - } - locals.clear(); - // Put it back so we reuse the heap of the storage - self.ecx.machine.written_only_inside_own_block_locals = locals; - if cfg!(debug_assertions) { - // Ensure we are correctly erasing locals with the non-debug-assert logic. - for local in self.ecx.machine.only_propagate_inside_block_locals.iter() { - assert!( - self.get_const(local.into()).is_none() - || self - .layout_of(self.local_decls[local].ty) - .map_or(true, |layout| layout.is_zst()) - ) + let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); + for (local, &mode) in can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(local), + ConstPropMode::OnlyInsideOwnBlock => { + Self::remove_const(&mut self.ecx, local); + self.ensure_not_propagated(local); + } } } + self.ecx.machine.can_const_prop = can_const_prop; } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 77402b8737e..68e50070e56 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -1,24 +1,17 @@ //! Propagates constants for early reporting of statically known //! assertion failures -use std::cell::Cell; - use either::{Left, Right}; use rustc_const_eval::interpret::Immediate; use rustc_const_eval::interpret::{ - self, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, + self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, }; use rustc_hir::def::DefKind; use rustc_hir::HirId; -use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{ - AssertKind, BinOp, Body, Constant, Local, LocalDecl, Location, Operand, Place, Rvalue, - SourceInfo, SourceScope, SourceScopeData, Statement, StatementKind, Terminator, TerminatorKind, - UnOp, RETURN_PLACE, -}; +use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; use rustc_middle::ty::InternalSubsts; use rustc_middle::ty::{ @@ -185,17 +178,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let param_env = tcx.param_env_reveal_all_normalized(def_id); let can_const_prop = CanConstProp::check(tcx, param_env, body); - let mut only_propagate_inside_block_locals = BitSet::new_empty(can_const_prop.len()); - for (l, mode) in can_const_prop.iter_enumerated() { - if *mode == ConstPropMode::OnlyInsideOwnBlock { - only_propagate_inside_block_locals.insert(l); - } - } let mut ecx = InterpCx::new( tcx, tcx.def_span(def_id), param_env, - ConstPropMachine::new(only_propagate_inside_block_locals, can_const_prop), + ConstPropMachine::new(can_const_prop), ); let ret_layout = ecx @@ -258,10 +245,8 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local] = LocalState { - value: LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)), - layout: Cell::new(None), - }; + ecx.frame_mut().locals[local].value = + LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); } fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { @@ -420,12 +405,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Some(()) } - fn const_prop( - &mut self, - rvalue: &Rvalue<'tcx>, - source_info: SourceInfo, - place: Place<'tcx>, - ) -> Option<()> { + fn check_rvalue(&mut self, rvalue: &Rvalue<'tcx>, source_info: SourceInfo) -> Option<()> { // Perform any special handling for specific Rvalue types. // Generally, checks here fall into one of two categories: // 1. Additional checking to provide useful lints to the user @@ -501,7 +481,20 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return None; } - self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place)) + Some(()) + } + + fn ensure_not_propagated(&mut self, local: Local) { + if cfg!(debug_assertions) { + assert!( + self.get_const(local.into()).is_none() + || self + .layout_of(self.local_decls[local].ty) + .map_or(true, |layout| layout.is_zst()), + "failed to remove values for `{local:?}`, value={:?}", + self.get_const(local.into()), + ) + } } } @@ -522,82 +515,78 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { self.eval_constant(constant, self.source_info.unwrap()); } + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + self.super_assign(place, rvalue, location); + + let source_info = self.source_info.unwrap(); + let Some(()) = self.check_rvalue(rvalue, source_info) else { return }; + + match self.ecx.machine.can_const_prop[place.local] { + // Do nothing if the place is indirect. + _ if place.is_indirect() => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), + ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => { + if self + .use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, *place)) + .is_none() + { + // Const prop failed, so erase the destination, ensuring that whatever happens + // from here on, does not know about the previous value. + // This is important in case we have + // ```rust + // let mut x = 42; + // x = SOME_MUTABLE_STATIC; + // // x must now be uninit + // ``` + // FIXME: we overzealously erase the entire local, because that's easier to + // implement. + trace!( + "propagation into {:?} failed. + Nuking the entire site from orbit, it's the only way to be sure", + place, + ); + Self::remove_const(&mut self.ecx, place.local); + } + } + } + } + fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { trace!("visit_statement: {:?}", statement); let source_info = statement.source_info; self.source_info = Some(source_info); - if let StatementKind::Assign(box (place, ref rval)) = statement.kind { - let can_const_prop = self.ecx.machine.can_const_prop[place.local]; - if let Some(()) = self.const_prop(rval, source_info, place) { - match can_const_prop { - ConstPropMode::OnlyInsideOwnBlock => { - trace!( - "found local restricted to its block. \ - Will remove it from const-prop after block is finished. Local: {:?}", - place.local - ); - } - ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { - trace!("can't propagate into {:?}", place); - if place.local != RETURN_PLACE { + + // We want to evaluate operands before any change to the assigned-to value, + // so we recurse first. + self.super_statement(statement, location); + + match statement.kind { + StatementKind::SetDiscriminant { ref place, .. } => { + match self.ecx.machine.can_const_prop[place.local] { + // Do nothing if the place is indirect. + _ if place.is_indirect() => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), + ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { + if self.use_ecx(source_info, |this| this.ecx.statement(statement)).is_some() + { + trace!("propped discriminant into {:?}", place); + } else { Self::remove_const(&mut self.ecx, place.local); } } - ConstPropMode::FullConstProp => {} } - } else { - // Const prop failed, so erase the destination, ensuring that whatever happens - // from here on, does not know about the previous value. - // This is important in case we have - // ```rust - // let mut x = 42; - // x = SOME_MUTABLE_STATIC; - // // x must now be uninit - // ``` - // FIXME: we overzealously erase the entire local, because that's easier to - // implement. - trace!( - "propagation into {:?} failed. - Nuking the entire site from orbit, it's the only way to be sure", - place, - ); - Self::remove_const(&mut self.ecx, place.local); } - } else { - match statement.kind { - StatementKind::SetDiscriminant { ref place, .. } => { - match self.ecx.machine.can_const_prop[place.local] { - ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { - if self - .use_ecx(source_info, |this| this.ecx.statement(statement)) - .is_some() - { - trace!("propped discriminant into {:?}", place); - } else { - Self::remove_const(&mut self.ecx, place.local); - } - } - ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => { - Self::remove_const(&mut self.ecx, place.local); - } - } - } - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].value = - if let StatementKind::StorageLive(_) = statement.kind { - LocalValue::Live(interpret::Operand::Immediate( - interpret::Immediate::Uninit, - )) - } else { - LocalValue::Dead - }; - } - _ => {} + StatementKind::StorageLive(local) => { + let frame = self.ecx.frame_mut(); + frame.locals[local].value = + LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + } + StatementKind::StorageDead(local) => { + let frame = self.ecx.frame_mut(); + frame.locals[local].value = LocalValue::Dead; } + _ => {} } - - self.super_statement(statement, location); } fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { @@ -694,27 +683,25 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { | TerminatorKind::Call { .. } | TerminatorKind::InlineAsm { .. } => {} } + } + + fn visit_basic_block_data(&mut self, block: BasicBlock, data: &BasicBlockData<'tcx>) { + self.super_basic_block_data(block, data); // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let mut locals = std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); - for &local in locals.iter() { - Self::remove_const(&mut self.ecx, local); - } - locals.clear(); - // Put it back so we reuse the heap of the storage - self.ecx.machine.written_only_inside_own_block_locals = locals; - if cfg!(debug_assertions) { - // Ensure we are correctly erasing locals with the non-debug-assert logic. - for local in self.ecx.machine.only_propagate_inside_block_locals.iter() { - assert!( - self.get_const(local.into()).is_none() - || self - .layout_of(self.local_decls[local].ty) - .map_or(true, |layout| layout.is_zst()) - ) + let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); + for (local, &mode) in can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation => self.ensure_not_propagated(local), + ConstPropMode::OnlyInsideOwnBlock => { + Self::remove_const(&mut self.ecx, local); + self.ensure_not_propagated(local); + } } } + self.ecx.machine.can_const_prop = can_const_prop; } } diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index c03cacd8615..49028ca4e5e 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -537,29 +537,29 @@ impl TraverseCoverageGraphWithLoops { "TraverseCoverageGraphWithLoops::next - context_stack: {:?}", self.context_stack.iter().rev().collect::<Vec<_>>() ); - while let Some(next_bcb) = { - // Strip contexts with empty worklists from the top of the stack - while self.context_stack.last().map_or(false, |context| context.worklist.is_empty()) { + + while let Some(context) = self.context_stack.last_mut() { + if let Some(next_bcb) = context.worklist.pop() { + if !self.visited.insert(next_bcb) { + debug!("Already visited: {:?}", next_bcb); + continue; + } + debug!("Visiting {:?}", next_bcb); + if self.backedges[next_bcb].len() > 0 { + debug!("{:?} is a loop header! Start a new TraversalContext...", next_bcb); + self.context_stack.push(TraversalContext { + loop_backedges: Some((self.backedges[next_bcb].clone(), next_bcb)), + worklist: Vec::new(), + }); + } + self.extend_worklist(basic_coverage_blocks, next_bcb); + return Some(next_bcb); + } else { + // Strip contexts with empty worklists from the top of the stack self.context_stack.pop(); } - // Pop the next bcb off of the current context_stack. If none, all BCBs were visited. - self.context_stack.last_mut().map_or(None, |context| context.worklist.pop()) - } { - if !self.visited.insert(next_bcb) { - debug!("Already visited: {:?}", next_bcb); - continue; - } - debug!("Visiting {:?}", next_bcb); - if self.backedges[next_bcb].len() > 0 { - debug!("{:?} is a loop header! Start a new TraversalContext...", next_bcb); - self.context_stack.push(TraversalContext { - loop_backedges: Some((self.backedges[next_bcb].clone(), next_bcb)), - worklist: Vec::new(), - }); - } - self.extend_worklist(basic_coverage_blocks, next_bcb); - return Some(next_bcb); } + None } diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 9a617159813..5ecb2d6a631 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -577,5 +577,5 @@ fn get_body_span<'tcx>( fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 { // FIXME(cjgillot) Stop hashing HIR manually here. let owner = hir_body.id().hir_id.owner; - tcx.hir_owner_nodes(owner).unwrap().hash_including_bodies.to_smaller_hash() + tcx.hir_owner_nodes(owner).unwrap().opt_hash_including_bodies.unwrap().to_smaller_hash() } diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index 7508df92df1..b8a5b92be4a 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -40,7 +40,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { let temp = self.patcher.new_internal_with_info( ty, self.local_decls[p_ref.local].source_info.span, - Some(Box::new(LocalInfo::DerefTemp)), + LocalInfo::DerefTemp, ); // We are adding current p_ref's projections to our diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 7344ec793ea..35e7efed87a 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -788,7 +788,7 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> { fn is_local_required(local: Local, body: &Body<'_>) -> bool { match body.local_kind(local) { LocalKind::Arg | LocalKind::ReturnPointer => true, - LocalKind::Var | LocalKind::Temp => false, + LocalKind::Temp => false, } } diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index b7f1cdfc7f2..e6875fad306 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -924,13 +924,19 @@ fn compute_layout<'tcx>( debug!(?decl); let ignore_for_traits = if tcx.sess.opts.unstable_opts.drop_tracking_mir { + // Do not `assert_crate_local` here, as post-borrowck cleanup may have already cleared + // the information. This is alright, since `ignore_for_traits` is only relevant when + // this code runs on pre-cleanup MIR, and `ignore_for_traits = false` is the safer + // default. match decl.local_info { // Do not include raw pointers created from accessing `static` items, as those could // well be re-created by another access to the same static. - Some(box LocalInfo::StaticRef { is_thread_local, .. }) => !is_thread_local, + ClearCrossCrate::Set(box LocalInfo::StaticRef { is_thread_local, .. }) => { + !is_thread_local + } // Fake borrows are only read by fake reads, so do not have any reality in // post-analysis MIR. - Some(box LocalInfo::FakeBorrow) => true, + ClearCrossCrate::Set(box LocalInfo::FakeBorrow) => true, _ => false, } } else { diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 89e0a007dac..9447a2ff040 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -158,10 +158,12 @@ impl EnumSizeOpt { tmp_ty, ), }; - let rval = Rvalue::Use(Operand::Constant(box (constant_vals))); + let rval = Rvalue::Use(Operand::Constant(Box::new(constant_vals))); - let const_assign = - Statement { source_info, kind: StatementKind::Assign(box (place, rval)) }; + let const_assign = Statement { + source_info, + kind: StatementKind::Assign(Box::new((place, rval))), + }; let discr_place = Place::from( local_decls @@ -170,7 +172,10 @@ impl EnumSizeOpt { let store_discr = Statement { source_info, - kind: StatementKind::Assign(box (discr_place, Rvalue::Discriminant(*rhs))), + kind: StatementKind::Assign(Box::new(( + discr_place, + Rvalue::Discriminant(*rhs), + ))), }; let discr_cast_place = @@ -178,14 +183,14 @@ impl EnumSizeOpt { let cast_discr = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( discr_cast_place, Rvalue::Cast( CastKind::IntToInt, Operand::Copy(discr_place), tcx.types.usize, ), - )), + ))), }; let size_place = @@ -193,14 +198,14 @@ impl EnumSizeOpt { let store_size = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( size_place, Rvalue::Use(Operand::Copy(Place { local: size_array_local, projection: tcx .mk_place_elems(&[PlaceElem::Index(discr_cast_place.local)]), })), - )), + ))), }; let dst = @@ -208,10 +213,10 @@ impl EnumSizeOpt { let dst_ptr = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( dst, Rvalue::AddressOf(Mutability::Mut, *lhs), - )), + ))), }; let dst_cast_ty = tcx.mk_mut_ptr(tcx.types.u8); @@ -220,10 +225,10 @@ impl EnumSizeOpt { let dst_cast = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( dst_cast_place, Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(dst), dst_cast_ty), - )), + ))), }; let src = @@ -231,10 +236,10 @@ impl EnumSizeOpt { let src_ptr = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( src, Rvalue::AddressOf(Mutability::Not, *rhs), - )), + ))), }; let src_cast_ty = tcx.mk_imm_ptr(tcx.types.u8); @@ -243,24 +248,24 @@ impl EnumSizeOpt { let src_cast = Statement { source_info, - kind: StatementKind::Assign(box ( + kind: StatementKind::Assign(Box::new(( src_cast_place, Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(src), src_cast_ty), - )), + ))), }; let deinit_old = - Statement { source_info, kind: StatementKind::Deinit(box dst) }; + Statement { source_info, kind: StatementKind::Deinit(Box::new(dst)) }; let copy_bytes = Statement { source_info, - kind: StatementKind::Intrinsic( - box NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { + kind: StatementKind::Intrinsic(Box::new( + NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { src: Operand::Copy(src_cast_place), dst: Operand::Copy(dst_cast_place), count: Operand::Copy(size_place), }), - ), + )), }; let store_dead = Statement { diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 5fd923190ef..50c3023b02b 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,7 +1,6 @@ #![allow(rustc::potential_query_instability)] #![feature(box_patterns)] #![feature(drain_filter)] -#![feature(box_syntax)] #![feature(let_chains)] #![feature(map_try_insert)] #![feature(min_specialization)] @@ -30,9 +29,9 @@ use rustc_hir::intravisit::{self, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; use rustc_middle::mir::{ - traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand, - Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, - TerminatorKind, + traversal, AnalysisPhase, Body, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, MirPass, + MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, + Statement, StatementKind, TerminatorKind, }; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; @@ -278,14 +277,14 @@ fn mir_const(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Steal< // Unsafety check uses the raw mir, so make sure it is run. if !tcx.sess.opts.unstable_opts.thir_unsafeck { if let Some(param_did) = def.const_param_did { - tcx.ensure().unsafety_check_result_for_const_arg((def.did, param_did)); + tcx.ensure_with_value().unsafety_check_result_for_const_arg((def.did, param_did)); } else { - tcx.ensure().unsafety_check_result(def.did); + tcx.ensure_with_value().unsafety_check_result(def.did); } } // has_ffi_unwind_calls query uses the raw mir, so make sure it is run. - tcx.ensure().has_ffi_unwind_calls(def.did); + tcx.ensure_with_value().has_ffi_unwind_calls(def.did); let mut body = tcx.mir_built(def).steal(); @@ -433,7 +432,7 @@ fn mir_drops_elaborated_and_const_checked( if tcx.sess.opts.unstable_opts.drop_tracking_mir && let DefKind::Generator = tcx.def_kind(def.did) { - tcx.ensure().mir_generator_witnesses(def.did); + tcx.ensure_with_value().mir_generator_witnesses(def.did); } let mir_borrowck = tcx.mir_borrowck_opt_const_arg(def); @@ -444,7 +443,7 @@ fn mir_drops_elaborated_and_const_checked( // Do not compute the mir call graph without said call graph actually being used. if inline::Inline.is_enabled(&tcx.sess) { - let _ = tcx.mir_inliner_callees(ty::InstanceDef::Item(def)); + tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def)); } } @@ -533,6 +532,12 @@ fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &[&lower_intrinsics::LowerIntrinsics, &simplify::SimplifyCfg::new("elaborate-drops")]; pm::run_passes(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::PostCleanup))); + + // Clear this by anticipation. Optimizations and runtime MIR have no reason to look + // into this information, which is meant for borrowck diagnostics. + for decl in &mut body.local_decls { + decl.local_info = ClearCrossCrate::Clear; + } } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -613,7 +618,7 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { // Run the `mir_for_ctfe` query, which depends on `mir_drops_elaborated_and_const_checked` // which we are going to steal below. Thus we need to run `mir_for_ctfe` first, so it // computes and caches its result. - Some(hir::ConstContext::ConstFn) => tcx.ensure().mir_for_ctfe(did), + Some(hir::ConstContext::ConstFn) => tcx.ensure_with_value().mir_for_ctfe(did), None => {} Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), } diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 4291e81c78c..b6e73eaad50 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -102,7 +102,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { mir::LocalKind::Arg => return None, mir::LocalKind::ReturnPointer => bug!("Return place was assigned to itself?"), - mir::LocalKind::Var | mir::LocalKind::Temp => {} + mir::LocalKind::Temp => {} } // If multiple different locals are copied to the return place. We can't pick a |