diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop.rs | 167 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop_lint.rs | 512 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/dataflow_const_prop.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/errors.rs | 42 |
4 files changed, 362 insertions, 366 deletions
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index c5824c30770..8b9e507c8c7 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -1,21 +1,12 @@ //! Propagates constants for early reporting of statically known //! assertion failures -use rustc_const_eval::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, FnArg, Frame, ImmTy, InterpCx, - InterpResult, OpTy, PlaceTy, Pointer, -}; -use rustc_data_structures::fx::FxHashSet; use rustc_index::bit_set::BitSet; use rustc_index::IndexVec; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::query::TyCtxtAt; -use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{self, ParamEnv, TyCtxt}; -use rustc_span::def_id::DefId; +use rustc_middle::ty::{ParamEnv, TyCtxt}; use rustc_target::abi::Size; -use rustc_target::spec::abi::Abi as CallAbi; /// The maximum number of bytes that we'll allocate space for a local or the return value. /// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just @@ -49,162 +40,6 @@ pub(crate) macro throw_machine_stop_str($($tt:tt)*) {{ throw_machine_stop!(Zst) }} -pub(crate) struct ConstPropMachine<'mir, 'tcx> { - /// The virtual call stack. - stack: Vec<Frame<'mir, 'tcx>>, - pub written_only_inside_own_block_locals: FxHashSet<Local>, - pub can_const_prop: IndexVec<Local, ConstPropMode>, -} - -impl ConstPropMachine<'_, '_> { - pub fn new(can_const_prop: IndexVec<Local, ConstPropMode>) -> Self { - Self { - stack: Vec::new(), - written_only_inside_own_block_locals: Default::default(), - can_const_prop, - } - } -} - -impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { - compile_time_machine!(<'mir, 'tcx>); - - const PANIC_ON_ALLOC_FAIL: bool = true; // all allocations are small (see `MAX_ALLOC_LIMIT`) - - const POST_MONO_CHECKS: bool = false; // this MIR is still generic! - - type MemoryKind = !; - - #[inline(always)] - fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { - false // no reason to enforce alignment - } - - #[inline(always)] - fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool { - false // for now, we don't enforce validity - } - - fn load_mir( - _ecx: &InterpCx<'mir, 'tcx, Self>, - _instance: ty::InstanceDef<'tcx>, - ) -> InterpResult<'tcx, &'tcx Body<'tcx>> { - throw_machine_stop_str!("calling functions isn't supported in ConstProp") - } - - fn panic_nounwind(_ecx: &mut InterpCx<'mir, 'tcx, Self>, _msg: &str) -> InterpResult<'tcx> { - throw_machine_stop_str!("panicking isn't supported in ConstProp") - } - - fn find_mir_or_eval_fn( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _instance: ty::Instance<'tcx>, - _abi: CallAbi, - _args: &[FnArg<'tcx>], - _destination: &PlaceTy<'tcx>, - _target: Option<BasicBlock>, - _unwind: UnwindAction, - ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> { - Ok(None) - } - - fn call_intrinsic( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _instance: ty::Instance<'tcx>, - _args: &[OpTy<'tcx>], - _destination: &PlaceTy<'tcx>, - _target: Option<BasicBlock>, - _unwind: UnwindAction, - ) -> InterpResult<'tcx> { - throw_machine_stop_str!("calling intrinsics isn't supported in ConstProp") - } - - fn assert_panic( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _msg: &rustc_middle::mir::AssertMessage<'tcx>, - _unwind: rustc_middle::mir::UnwindAction, - ) -> InterpResult<'tcx> { - bug!("panics terminators are not evaluated in ConstProp") - } - - fn binary_ptr_op( - _ecx: &InterpCx<'mir, 'tcx, Self>, - _bin_op: BinOp, - _left: &ImmTy<'tcx>, - _right: &ImmTy<'tcx>, - ) -> InterpResult<'tcx, (ImmTy<'tcx>, bool)> { - // We can't do this because aliasing of memory can differ between const eval and llvm - throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") - } - - fn before_access_local_mut<'a>( - ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - frame: usize, - local: Local, - ) -> InterpResult<'tcx> { - assert_eq!(frame, 0); - match ecx.machine.can_const_prop[local] { - ConstPropMode::NoPropagation => { - throw_machine_stop_str!( - "tried to write to a local that is marked as not propagatable" - ) - } - ConstPropMode::OnlyInsideOwnBlock => { - ecx.machine.written_only_inside_own_block_locals.insert(local); - } - ConstPropMode::FullConstProp => {} - } - Ok(()) - } - - fn before_access_global( - _tcx: TyCtxtAt<'tcx>, - _machine: &Self, - _alloc_id: AllocId, - alloc: ConstAllocation<'tcx>, - _static_def_id: Option<DefId>, - is_write: bool, - ) -> InterpResult<'tcx> { - if is_write { - throw_machine_stop_str!("can't write to global"); - } - // If the static allocation is mutable, then we can't const prop it as its content - // might be different at runtime. - if alloc.inner().mutability.is_mut() { - throw_machine_stop_str!("can't access mutable globals in ConstProp"); - } - - Ok(()) - } - - #[inline(always)] - fn expose_ptr(_ecx: &mut InterpCx<'mir, 'tcx, Self>, _ptr: Pointer) -> InterpResult<'tcx> { - throw_machine_stop_str!("exposing pointers isn't supported in ConstProp") - } - - #[inline(always)] - fn init_frame_extra( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - frame: Frame<'mir, 'tcx>, - ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { - Ok(frame) - } - - #[inline(always)] - fn stack<'a>( - ecx: &'a InterpCx<'mir, 'tcx, Self>, - ) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] { - &ecx.machine.stack - } - - #[inline(always)] - fn stack_mut<'a>( - ecx: &'a mut InterpCx<'mir, 'tcx, Self>, - ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>> { - &mut ecx.machine.stack - } -} - /// The mode that `ConstProp` is allowed to run in for a given `Local`. #[derive(Clone, Copy, Debug, PartialEq)] pub enum ConstPropMode { diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index b8fecaf635a..aa22b8c7c58 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -3,37 +3,26 @@ use std::fmt::Debug; -use either::Left; - -use rustc_const_eval::interpret::Immediate; -use rustc_const_eval::interpret::{ - InterpCx, InterpResult, MemoryKind, OpTy, Scalar, StackPopCleanup, -}; -use rustc_const_eval::ReportErrorExt; +use rustc_const_eval::interpret::{ImmTy, Projectable}; +use rustc_const_eval::interpret::{InterpCx, InterpResult, Scalar}; +use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_hir::HirId; use rustc_index::bit_set::BitSet; +use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; -use rustc_middle::ty::GenericArgs; -use rustc_middle::ty::{ - self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt, -}; +use rustc_middle::ty::{self, ConstInt, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt}; use rustc_span::Span; -use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; +use rustc_target::abi::{Abi, FieldIdx, HasDataLayout, Size, TargetDataLayout, VariantIdx}; use crate::const_prop::CanConstProp; -use crate::const_prop::ConstPropMachine; use crate::const_prop::ConstPropMode; -use crate::errors::AssertLint; +use crate::dataflow_const_prop::DummyMachine; +use crate::errors::{AssertLint, AssertLintKind}; use crate::MirLint; -/// The maximum number of bytes that we'll allocate space for a local or the return value. -/// Needed for #66397, because otherwise we eval into large places and that can cause OOM or just -/// Severely regress performance. -const MAX_ALLOC_LIMIT: u64 = 1024; - pub struct ConstPropLint; impl<'tcx> MirLint<'tcx> for ConstPropLint { @@ -81,11 +70,85 @@ impl<'tcx> MirLint<'tcx> for ConstPropLint { /// Finds optimization opportunities on the MIR. struct ConstPropagator<'mir, 'tcx> { - ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, + ecx: InterpCx<'mir, 'tcx, DummyMachine>, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, worklist: Vec<BasicBlock>, visited_blocks: BitSet<BasicBlock>, + locals: IndexVec<Local, Value<'tcx>>, + body: &'mir Body<'tcx>, + written_only_inside_own_block_locals: FxHashSet<Local>, + can_const_prop: IndexVec<Local, ConstPropMode>, +} + +#[derive(Debug, Clone)] +enum Value<'tcx> { + Immediate(ImmTy<'tcx>), + Aggregate { variant: VariantIdx, fields: IndexVec<FieldIdx, Value<'tcx>> }, + Uninit, +} + +impl<'tcx> From<ImmTy<'tcx>> for Value<'tcx> { + fn from(v: ImmTy<'tcx>) -> Self { + Self::Immediate(v) + } +} + +impl<'tcx> Value<'tcx> { + fn project( + &self, + proj: &[PlaceElem<'tcx>], + prop: &ConstPropagator<'_, 'tcx>, + ) -> Option<&Value<'tcx>> { + let mut this = self; + for proj in proj { + this = match (*proj, this) { + (PlaceElem::Field(idx, _), Value::Aggregate { fields, .. }) => { + fields.get(idx).unwrap_or(&Value::Uninit) + } + (PlaceElem::Index(idx), Value::Aggregate { fields, .. }) => { + let idx = prop.get_const(idx.into())?.immediate()?; + let idx = prop.ecx.read_target_usize(idx).ok()?; + fields.get(FieldIdx::from_u32(idx.try_into().ok()?)).unwrap_or(&Value::Uninit) + } + ( + PlaceElem::ConstantIndex { offset, min_length: _, from_end: false }, + Value::Aggregate { fields, .. }, + ) => fields + .get(FieldIdx::from_u32(offset.try_into().ok()?)) + .unwrap_or(&Value::Uninit), + _ => return None, + }; + } + Some(this) + } + + fn project_mut(&mut self, proj: &[PlaceElem<'_>]) -> Option<&mut Value<'tcx>> { + let mut this = self; + for proj in proj { + this = match (proj, this) { + (PlaceElem::Field(idx, _), Value::Aggregate { fields, .. }) => { + fields.ensure_contains_elem(*idx, || Value::Uninit) + } + (PlaceElem::Field(..), val @ Value::Uninit) => { + *val = Value::Aggregate { + variant: VariantIdx::new(0), + fields: Default::default(), + }; + val.project_mut(&[*proj])? + } + _ => return None, + }; + } + Some(this) + } + + fn immediate(&self) -> Option<&ImmTy<'tcx>> { + match self { + Value::Immediate(op) => Some(op), + _ => None, + } + } } impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { @@ -121,49 +184,10 @@ impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn new(body: &'mir Body<'tcx>, tcx: TyCtxt<'tcx>) -> ConstPropagator<'mir, 'tcx> { let def_id = body.source.def_id(); - let args = &GenericArgs::identity_for_item(tcx, def_id); let param_env = tcx.param_env_reveal_all_normalized(def_id); let can_const_prop = CanConstProp::check(tcx, param_env, body); - let mut ecx = InterpCx::new( - tcx, - tcx.def_span(def_id), - param_env, - ConstPropMachine::new(can_const_prop), - ); - - let ret_layout = ecx - .layout_of(body.bound_return_ty().instantiate(tcx, args)) - .ok() - // Don't bother allocating memory for large values. - // I don't know how return types can seem to be unsized but this happens in the - // `type/type-unsatisfiable.rs` test. - .filter(|ret_layout| { - ret_layout.is_sized() && ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) - }) - .unwrap_or_else(|| ecx.layout_of(tcx.types.unit).unwrap()); - - let ret = ecx - .allocate(ret_layout, MemoryKind::Stack) - .expect("couldn't perform small allocation") - .into(); - - ecx.push_stack_frame( - Instance::new(def_id, args), - body, - &ret, - StackPopCleanup::Root { cleanup: false }, - ) - .expect("failed to push initial stack frame"); - - for local in body.local_decls.indices() { - // Mark everything initially live. - // This is somewhat dicey since some of them might be unsized and it is incoherent to - // mark those as live... We rely on `local_to_place`/`local_to_op` in the interpreter - // stopping us before those unsized immediates can cause issues deeper in the - // interpreter. - ecx.frame_mut().locals[local].make_live_uninit(); - } + let ecx = InterpCx::new(tcx, tcx.def_span(def_id), param_env, DummyMachine); ConstPropagator { ecx, @@ -171,61 +195,47 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { param_env, worklist: vec![START_BLOCK], visited_blocks: BitSet::new_empty(body.basic_blocks.len()), + locals: IndexVec::from_elem_n(Value::Uninit, body.local_decls.len()), + body, + can_const_prop, + written_only_inside_own_block_locals: Default::default(), } } - fn body(&self) -> &'mir Body<'tcx> { - self.ecx.frame().body - } - fn local_decls(&self) -> &'mir LocalDecls<'tcx> { - &self.body().local_decls + &self.body.local_decls } - fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> { - let op = match self.ecx.eval_place_to_op(place, None) { - Ok(op) => { - if op - .as_mplace_or_imm() - .right() - .is_some_and(|imm| matches!(*imm, Immediate::Uninit)) - { - // Make sure nobody accidentally uses this value. - return None; - } - op - } - Err(e) => { - trace!("get_const failed: {:?}", e.into_kind().debug()); - return None; - } - }; - - // Try to read the local as an immediate so that if it is representable as a scalar, we can - // handle it as such, but otherwise, just return the value as is. - Some(match self.ecx.read_immediate_raw(&op) { - Ok(Left(imm)) => imm.into(), - _ => op, - }) + fn get_const(&self, place: Place<'tcx>) -> Option<&Value<'tcx>> { + self.locals[place.local].project(&place.projection, self) } /// Remove `local` from the pool of `Locals`. Allows writing to them, /// but not reading from them anymore. - fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { - ecx.frame_mut().locals[local].make_live_uninit(); - ecx.machine.written_only_inside_own_block_locals.remove(&local); + fn remove_const(&mut self, local: Local) { + self.locals[local] = Value::Uninit; + self.written_only_inside_own_block_locals.remove(&local); + } + + fn access_mut(&mut self, place: &Place<'_>) -> Option<&mut Value<'tcx>> { + match self.can_const_prop[place.local] { + ConstPropMode::NoPropagation => return None, + ConstPropMode::OnlyInsideOwnBlock => { + self.written_only_inside_own_block_locals.insert(place.local); + } + ConstPropMode::FullConstProp => {} + } + self.locals[place.local].project_mut(place.projection) } fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { - source_info.scope.lint_root(&self.body().source_scopes) + source_info.scope.lint_root(&self.body.source_scopes) } - fn use_ecx<F, T>(&mut self, location: Location, f: F) -> Option<T> + fn use_ecx<F, T>(&mut self, f: F) -> Option<T> where F: FnOnce(&mut Self) -> InterpResult<'tcx, T>, { - // Overwrite the PC -- whatever the interpreter does to it does not make any sense anyway. - self.ecx.frame_mut().loc = Left(location); match f(self) { Ok(val) => Some(val), Err(error) => { @@ -244,7 +254,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } /// Returns the value, if any, of evaluating `c`. - fn eval_constant(&mut self, c: &ConstOperand<'tcx>, location: Location) -> Option<OpTy<'tcx>> { + fn eval_constant(&mut self, c: &ConstOperand<'tcx>) -> Option<ImmTy<'tcx>> { // FIXME we need to revisit this for #67176 if c.has_param() { return None; @@ -258,46 +268,62 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // manually normalized. let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.const_).ok()?; - self.use_ecx(location, |this| this.ecx.eval_mir_constant(&val, Some(c.span), None)) + self.use_ecx(|this| this.ecx.eval_mir_constant(&val, Some(c.span), None))? + .as_mplace_or_imm() + .right() } /// Returns the value, if any, of evaluating `place`. - fn eval_place(&mut self, place: Place<'tcx>, location: Location) -> Option<OpTy<'tcx>> { - trace!("eval_place(place={:?})", place); - self.use_ecx(location, |this| this.ecx.eval_place_to_op(place, None)) + #[instrument(level = "trace", skip(self), ret)] + fn eval_place(&mut self, place: Place<'tcx>) -> Option<ImmTy<'tcx>> { + match self.get_const(place)? { + Value::Immediate(imm) => Some(imm.clone()), + Value::Aggregate { .. } => None, + Value::Uninit => None, + } } /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant` /// or `eval_place`, depending on the variant of `Operand` used. - fn eval_operand(&mut self, op: &Operand<'tcx>, location: Location) -> Option<OpTy<'tcx>> { + fn eval_operand(&mut self, op: &Operand<'tcx>) -> Option<ImmTy<'tcx>> { match *op { - Operand::Constant(ref c) => self.eval_constant(c, location), - Operand::Move(place) | Operand::Copy(place) => self.eval_place(place, location), + Operand::Constant(ref c) => self.eval_constant(c), + Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), } } - fn report_assert_as_lint(&self, source_info: &SourceInfo, lint: AssertLint<impl Debug>) { + fn report_assert_as_lint( + &self, + location: Location, + lint_kind: AssertLintKind, + assert_kind: AssertKind<impl Debug>, + ) { + let source_info = self.body.source_info(location); if let Some(lint_root) = self.lint_root(*source_info) { - self.tcx.emit_node_span_lint(lint.lint(), lint_root, source_info.span, lint); + let span = source_info.span; + self.tcx.emit_node_span_lint( + lint_kind.lint(), + lint_root, + span, + AssertLint { span, assert_kind, lint_kind }, + ); } } fn check_unary_op(&mut self, op: UnOp, arg: &Operand<'tcx>, location: Location) -> Option<()> { - if let (val, true) = self.use_ecx(location, |this| { - let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?; + let arg = self.eval_operand(arg)?; + if let (val, true) = self.use_ecx(|this| { + let val = this.ecx.read_immediate(&arg)?; let (_res, overflow) = this.ecx.overflowing_unary_op(op, &val)?; Ok((val, overflow)) })? { // `AssertKind` only has an `OverflowNeg` variant, so make sure that is // appropriate to use. assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow"); - let source_info = self.body().source_info(location); self.report_assert_as_lint( - source_info, - AssertLint::ArithmeticOverflow( - source_info.span, - AssertKind::OverflowNeg(val.to_const_int()), - ), + location, + AssertLintKind::ArithmeticOverflow, + AssertKind::OverflowNeg(val.to_const_int()), ); return None; } @@ -312,11 +338,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { right: &Operand<'tcx>, location: Location, ) -> Option<()> { - let r = self.use_ecx(location, |this| { - this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?) - }); - let l = self - .use_ecx(location, |this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)); + let r = + self.eval_operand(right).and_then(|r| self.use_ecx(|this| this.ecx.read_immediate(&r))); + let l = + self.eval_operand(left).and_then(|l| self.use_ecx(|this| this.ecx.read_immediate(&l))); // Check for exceeding shifts *even if* we cannot evaluate the LHS. if matches!(op, BinOp::Shr | BinOp::Shl) { let r = r.clone()?; @@ -328,7 +353,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let r_bits = r.to_scalar().to_bits(right_size).ok(); if r_bits.is_some_and(|b| b >= left_size.bits() as u128) { debug!("check_binary_op: reporting assert for {:?}", location); - let source_info = self.body().source_info(location); let panic = AssertKind::Overflow( op, match l { @@ -342,27 +366,21 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { }, r.to_const_int(), ); - self.report_assert_as_lint( - source_info, - AssertLint::ArithmeticOverflow(source_info.span, panic), - ); + self.report_assert_as_lint(location, AssertLintKind::ArithmeticOverflow, panic); return None; } } if let (Some(l), Some(r)) = (l, r) { // The remaining operators are handled through `overflowing_binary_op`. - if self.use_ecx(location, |this| { + if self.use_ecx(|this| { let (_res, overflow) = this.ecx.overflowing_binary_op(op, &l, &r)?; Ok(overflow) })? { - let source_info = self.body().source_info(location); self.report_assert_as_lint( - source_info, - AssertLint::ArithmeticOverflow( - source_info.span, - AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), - ), + location, + AssertLintKind::ArithmeticOverflow, + AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), ); return None; } @@ -411,7 +429,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // value the local has right now. // Thus, all locals that have their reference taken // must not take part in propagation. - Self::remove_const(&mut self.ecx, place.local); + self.remove_const(place.local); return None; } @@ -453,17 +471,17 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { cond: &Operand<'tcx>, location: Location, ) -> Option<!> { - let value = &self.eval_operand(cond, location)?; + let value = &self.eval_operand(cond)?; trace!("assertion on {:?} should be {:?}", value, expected); let expected = Scalar::from_bool(expected); - let value_const = self.use_ecx(location, |this| this.ecx.read_scalar(value))?; + let value_const = self.use_ecx(|this| this.ecx.read_scalar(value))?; if expected != value_const { // Poison all places this operand references so that further code // doesn't use the invalid value if let Some(place) = cond.place() { - Self::remove_const(&mut self.ecx, place.local); + self.remove_const(place.local); } enum DbgVal<T> { @@ -481,7 +499,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let mut eval_to_int = |op| { // This can be `None` if the lhs wasn't const propagated and we just // triggered the assert on the value of the rhs. - self.eval_operand(op, location) + self.eval_operand(op) .and_then(|op| self.ecx.read_immediate(&op).ok()) .map_or(DbgVal::Underscore, |op| DbgVal::Val(op.to_const_int())) }; @@ -503,11 +521,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Need proper const propagator for these. _ => return None, }; - let source_info = self.body().source_info(location); - self.report_assert_as_lint( - source_info, - AssertLint::UnconditionalPanic(source_info.span, msg), - ); + self.report_assert_as_lint(location, AssertLintKind::UnconditionalPanic, msg); } None @@ -515,16 +529,176 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn ensure_not_propagated(&self, local: Local) { if cfg!(debug_assertions) { + let val = self.get_const(local.into()); assert!( - self.get_const(local.into()).is_none() + matches!(val, Some(Value::Uninit)) || self .layout_of(self.local_decls()[local].ty) .map_or(true, |layout| layout.is_zst()), - "failed to remove values for `{local:?}`, value={:?}", - self.get_const(local.into()), + "failed to remove values for `{local:?}`, value={val:?}", ) } } + + #[instrument(level = "trace", skip(self), ret)] + fn eval_rvalue( + &mut self, + rvalue: &Rvalue<'tcx>, + location: Location, + dest: &Place<'tcx>, + ) -> Option<()> { + if !dest.projection.is_empty() { + return None; + } + use rustc_middle::mir::Rvalue::*; + let layout = self.ecx.layout_of(dest.ty(self.body, self.tcx).ty).ok()?; + trace!(?layout); + + let val: Value<'_> = match *rvalue { + ThreadLocalRef(_) => return None, + + Use(ref operand) => self.eval_operand(operand)?.into(), + + CopyForDeref(place) => self.eval_place(place)?.into(), + + BinaryOp(bin_op, box (ref left, ref right)) => { + let left = self.eval_operand(left)?; + let left = self.use_ecx(|this| this.ecx.read_immediate(&left))?; + + let right = self.eval_operand(right)?; + let right = self.use_ecx(|this| this.ecx.read_immediate(&right))?; + + let val = + self.use_ecx(|this| this.ecx.wrapping_binary_op(bin_op, &left, &right))?; + val.into() + } + + CheckedBinaryOp(bin_op, box (ref left, ref right)) => { + let left = self.eval_operand(left)?; + let left = self.use_ecx(|this| this.ecx.read_immediate(&left))?; + + let right = self.eval_operand(right)?; + let right = self.use_ecx(|this| this.ecx.read_immediate(&right))?; + + let (val, overflowed) = + self.use_ecx(|this| this.ecx.overflowing_binary_op(bin_op, &left, &right))?; + let overflowed = ImmTy::from_bool(overflowed, self.tcx); + Value::Aggregate { + variant: VariantIdx::new(0), + fields: [Value::from(val), overflowed.into()].into_iter().collect(), + } + } + + UnaryOp(un_op, ref operand) => { + let operand = self.eval_operand(operand)?; + let val = self.use_ecx(|this| this.ecx.read_immediate(&operand))?; + + let val = self.use_ecx(|this| this.ecx.wrapping_unary_op(un_op, &val))?; + val.into() + } + + Aggregate(ref kind, ref fields) => Value::Aggregate { + fields: fields + .iter() + .map(|field| self.eval_operand(field).map_or(Value::Uninit, Value::Immediate)) + .collect(), + variant: match **kind { + AggregateKind::Adt(_, variant, _, _, _) => variant, + AggregateKind::Array(_) + | AggregateKind::Tuple + | AggregateKind::Closure(_, _) + | AggregateKind::Coroutine(_, _) => VariantIdx::new(0), + }, + }, + + Repeat(ref op, n) => { + trace!(?op, ?n); + return None; + } + + Len(place) => { + let len = match self.get_const(place)? { + Value::Immediate(src) => src.len(&self.ecx).ok()?, + Value::Aggregate { fields, .. } => fields.len() as u64, + Value::Uninit => match place.ty(self.local_decls(), self.tcx).ty.kind() { + ty::Array(_, n) => n.try_eval_target_usize(self.tcx, self.param_env)?, + _ => return None, + }, + }; + ImmTy::from_scalar(Scalar::from_target_usize(len, self), layout).into() + } + + Ref(..) | AddressOf(..) => return None, + + NullaryOp(ref null_op, ty) => { + let op_layout = self.use_ecx(|this| this.ecx.layout_of(ty))?; + let val = match null_op { + NullOp::SizeOf => op_layout.size.bytes(), + NullOp::AlignOf => op_layout.align.abi.bytes(), + NullOp::OffsetOf(fields) => { + op_layout.offset_of_subfield(self, fields.iter()).bytes() + } + }; + ImmTy::from_scalar(Scalar::from_target_usize(val, self), layout).into() + } + + ShallowInitBox(..) => return None, + + Cast(ref kind, ref value, to) => match kind { + CastKind::IntToInt | CastKind::IntToFloat => { + let value = self.eval_operand(value)?; + let value = self.ecx.read_immediate(&value).ok()?; + let to = self.ecx.layout_of(to).ok()?; + let res = self.ecx.int_to_int_or_float(&value, to).ok()?; + res.into() + } + CastKind::FloatToFloat | CastKind::FloatToInt => { + let value = self.eval_operand(value)?; + let value = self.ecx.read_immediate(&value).ok()?; + let to = self.ecx.layout_of(to).ok()?; + let res = self.ecx.float_to_float_or_int(&value, to).ok()?; + res.into() + } + CastKind::Transmute => { + let value = self.eval_operand(value)?; + let to = self.ecx.layout_of(to).ok()?; + // `offset` for immediates only supports scalar/scalar-pair ABIs, + // so bail out if the target is not one. + match (value.layout.abi, to.abi) { + (Abi::Scalar(..), Abi::Scalar(..)) => {} + (Abi::ScalarPair(..), Abi::ScalarPair(..)) => {} + _ => return None, + } + + value.offset(Size::ZERO, to, &self.ecx).ok()?.into() + } + _ => return None, + }, + + Discriminant(place) => { + let variant = match self.get_const(place)? { + Value::Immediate(op) => { + let op = op.clone(); + self.use_ecx(|this| this.ecx.read_discriminant(&op))? + } + Value::Aggregate { variant, .. } => *variant, + Value::Uninit => return None, + }; + let imm = self.use_ecx(|this| { + this.ecx.discriminant_for_variant( + place.ty(this.local_decls(), this.tcx).ty, + variant, + ) + })?; + imm.into() + } + }; + trace!(?val); + + *self.access_mut(dest)? = val; + + Some(()) + } } impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { @@ -546,7 +720,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { fn visit_constant(&mut self, constant: &ConstOperand<'tcx>, location: Location) { trace!("visit_constant: {:?}", constant); self.super_constant(constant, location); - self.eval_constant(constant, location); + self.eval_constant(constant); } fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { @@ -554,15 +728,12 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { let Some(()) = self.check_rvalue(rvalue, location) else { return }; - match self.ecx.machine.can_const_prop[place.local] { + match self.can_const_prop[place.local] { // Do nothing if the place is indirect. _ if place.is_indirect() => {} ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => { - if self - .use_ecx(location, |this| this.ecx.eval_rvalue_into_place(rvalue, *place)) - .is_none() - { + if self.eval_rvalue(rvalue, location, place).is_none() { // Const prop failed, so erase the destination, ensuring that whatever happens // from here on, does not know about the previous value. // This is important in case we have @@ -578,7 +749,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { Nuking the entire site from orbit, it's the only way to be sure", place, ); - Self::remove_const(&mut self.ecx, place.local); + self.remove_const(place.local); } } } @@ -592,28 +763,24 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { self.super_statement(statement, location); match statement.kind { - StatementKind::SetDiscriminant { ref place, .. } => { - match self.ecx.machine.can_const_prop[place.local] { + StatementKind::SetDiscriminant { ref place, variant_index } => { + match self.can_const_prop[place.local] { // Do nothing if the place is indirect. _ if place.is_indirect() => {} ConstPropMode::NoPropagation => self.ensure_not_propagated(place.local), ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => { - if self.use_ecx(location, |this| this.ecx.statement(statement)).is_some() { - trace!("propped discriminant into {:?}", place); - } else { - Self::remove_const(&mut self.ecx, place.local); + match self.access_mut(place) { + Some(Value::Aggregate { variant, .. }) => *variant = variant_index, + _ => self.remove_const(place.local), } } } } StatementKind::StorageLive(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].make_live_uninit(); + self.remove_const(local); } StatementKind::StorageDead(local) => { - let frame = self.ecx.frame_mut(); - // We don't actually track liveness, so the local remains live. But forget its value. - frame.locals[local].make_live_uninit(); + self.remove_const(local); } _ => {} } @@ -626,9 +793,8 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { self.check_assertion(*expected, msg, cond, location); } TerminatorKind::SwitchInt { ref discr, ref targets } => { - if let Some(ref value) = self.eval_operand(discr, location) - && let Some(value_const) = - self.use_ecx(location, |this| this.ecx.read_scalar(value)) + if let Some(ref value) = self.eval_operand(discr) + && let Some(value_const) = self.use_ecx(|this| this.ecx.read_scalar(value)) && let Ok(constant) = value_const.try_to_int() && let Ok(constant) = constant.to_bits(constant.size()) { @@ -665,7 +831,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. let mut written_only_inside_own_block_locals = - std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); + std::mem::take(&mut self.written_only_inside_own_block_locals); // This loop can get very hot for some bodies: it check each local in each bb. // To avoid this quadratic behaviour, we only clear the locals that were modified inside @@ -673,17 +839,13 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { // The order in which we remove consts does not matter. #[allow(rustc::potential_query_instability)] for local in written_only_inside_own_block_locals.drain() { - debug_assert_eq!( - self.ecx.machine.can_const_prop[local], - ConstPropMode::OnlyInsideOwnBlock - ); - Self::remove_const(&mut self.ecx, local); + debug_assert_eq!(self.can_const_prop[local], ConstPropMode::OnlyInsideOwnBlock); + self.remove_const(local); } - self.ecx.machine.written_only_inside_own_block_locals = - written_only_inside_own_block_locals; + self.written_only_inside_own_block_locals = written_only_inside_own_block_locals; if cfg!(debug_assertions) { - for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() { + for (local, &mode) in self.can_const_prop.iter_enumerated() { match mode { ConstPropMode::FullConstProp => {} ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => { diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index ad12bce9b02..d5f22b2cdbc 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -403,7 +403,12 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { operand, &mut |elem, op| match elem { TrackElem::Field(idx) => self.ecx.project_field(op, idx.as_usize()).ok(), - TrackElem::Variant(idx) => self.ecx.project_downcast(op, idx).ok(), + TrackElem::Variant(idx) => { + if op.layout.for_variant(&self.ecx, idx).abi.is_uninhabited() { + return None; + } + self.ecx.project_downcast(op, idx).ok() + } TrackElem::Discriminant => { let variant = self.ecx.read_discriminant(op).ok()?; let discr_value = diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 2ee660ddc9b..4574cb4d28d 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -201,45 +201,39 @@ impl<'a> DecorateLint<'a, ()> for UnsafeOpInUnsafeFn { } } -pub(crate) enum AssertLint<P> { - ArithmeticOverflow(Span, AssertKind<P>), - UnconditionalPanic(Span, AssertKind<P>), +pub(crate) struct AssertLint<P> { + pub span: Span, + pub assert_kind: AssertKind<P>, + pub lint_kind: AssertLintKind, +} + +pub(crate) enum AssertLintKind { + ArithmeticOverflow, + UnconditionalPanic, } impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> { fn decorate_lint<'b>(self, diag: &'b mut DiagnosticBuilder<'a, ()>) { - let span = self.span(); - let assert_kind = self.panic(); - let message = assert_kind.diagnostic_message(); - assert_kind.add_args(&mut |name, value| { + let message = self.assert_kind.diagnostic_message(); + self.assert_kind.add_args(&mut |name, value| { diag.arg(name, value); }); - diag.span_label(span, message); + diag.span_label(self.span, message); } fn msg(&self) -> DiagnosticMessage { - match self { - AssertLint::ArithmeticOverflow(..) => fluent::mir_transform_arithmetic_overflow, - AssertLint::UnconditionalPanic(..) => fluent::mir_transform_operation_will_panic, + match self.lint_kind { + AssertLintKind::ArithmeticOverflow => fluent::mir_transform_arithmetic_overflow, + AssertLintKind::UnconditionalPanic => fluent::mir_transform_operation_will_panic, } } } -impl<P> AssertLint<P> { +impl AssertLintKind { pub fn lint(&self) -> &'static Lint { match self { - AssertLint::ArithmeticOverflow(..) => lint::builtin::ARITHMETIC_OVERFLOW, - AssertLint::UnconditionalPanic(..) => lint::builtin::UNCONDITIONAL_PANIC, - } - } - pub fn span(&self) -> Span { - match self { - AssertLint::ArithmeticOverflow(sp, _) | AssertLint::UnconditionalPanic(sp, _) => *sp, - } - } - pub fn panic(self) -> AssertKind<P> { - match self { - AssertLint::ArithmeticOverflow(_, p) | AssertLint::UnconditionalPanic(_, p) => p, + AssertLintKind::ArithmeticOverflow => lint::builtin::ARITHMETIC_OVERFLOW, + AssertLintKind::UnconditionalPanic => lint::builtin::UNCONDITIONAL_PANIC, } } } |