diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop.rs | 59 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/const_prop_lint.rs | 35 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/dataflow_const_prop.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/deduce_param_attrs.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/ffi_unwind_calls.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/generator.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/lib.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/lower_intrinsics.rs | 56 |
8 files changed, 144 insertions, 47 deletions
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index de7b8c63fc8..c1cf6ee0f9e 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -4,6 +4,7 @@ use either::Right; use rustc_const_eval::const_eval::CheckAlignment; +use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; use rustc_index::vec::IndexVec; @@ -151,12 +152,17 @@ impl<'tcx> MirPass<'tcx> for ConstProp { pub struct ConstPropMachine<'mir, 'tcx> { /// The virtual call stack. stack: Vec<Frame<'mir, 'tcx>>, + pub written_only_inside_own_block_locals: FxHashSet<Local>, pub can_const_prop: IndexVec<Local, ConstPropMode>, } impl ConstPropMachine<'_, '_> { pub fn new(can_const_prop: IndexVec<Local, ConstPropMode>) -> Self { - Self { stack: Vec::new(), can_const_prop } + Self { + stack: Vec::new(), + written_only_inside_own_block_locals: Default::default(), + can_const_prop, + } } } @@ -174,7 +180,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> } #[inline(always)] - fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool { false // for now, we don't enforce validity } fn alignment_check_failed( @@ -249,7 +255,10 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> "tried to write to a local that is marked as not propagatable" ) } - ConstPropMode::OnlyInsideOwnBlock | ConstPropMode::FullConstProp => {} + ConstPropMode::OnlyInsideOwnBlock => { + ecx.machine.written_only_inside_own_block_locals.insert(local); + } + ConstPropMode::FullConstProp => {} } ecx.machine.stack[frame].locals[local].access_mut() } @@ -416,6 +425,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { ecx.frame_mut().locals[local].value = LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.machine.written_only_inside_own_block_locals.remove(&local); } /// Returns the value, if any, of evaluating `c`. @@ -494,6 +504,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return None; } + // Do not try creating references, nor any types with potentially-complex + // invariants. This avoids an issue where checking validity would do a + // bunch of work generating a nice message about the invariant violation, + // only to not show it to anyone (since this isn't the lint). + Rvalue::Cast(CastKind::Transmute, op, dst_ty) if !dst_ty.is_primitive() => { + trace!("skipping Transmute of {:?} to {:?}", op, dst_ty); + + return None; + } // There's no other checking to do at this time. Rvalue::Aggregate(..) @@ -693,7 +712,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - fn ensure_not_propagated(&mut self, local: Local) { + fn ensure_not_propagated(&self, local: Local) { if cfg!(debug_assertions) { assert!( self.get_const(local.into()).is_none() @@ -963,17 +982,31 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); - for (local, &mode) in can_const_prop.iter_enumerated() { - match mode { - ConstPropMode::FullConstProp => {} - ConstPropMode::NoPropagation => self.ensure_not_propagated(local), - ConstPropMode::OnlyInsideOwnBlock => { - Self::remove_const(&mut self.ecx, local); - self.ensure_not_propagated(local); + let mut written_only_inside_own_block_locals = + std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); + + // This loop can get very hot for some bodies: it check each local in each bb. + // To avoid this quadratic behaviour, we only clear the locals that were modified inside + // the current block. + for local in written_only_inside_own_block_locals.drain() { + debug_assert_eq!( + self.ecx.machine.can_const_prop[local], + ConstPropMode::OnlyInsideOwnBlock + ); + Self::remove_const(&mut self.ecx, local); + } + self.ecx.machine.written_only_inside_own_block_locals = + written_only_inside_own_block_locals; + + if cfg!(debug_assertions) { + for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => { + self.ensure_not_propagated(local); + } } } } - self.ecx.machine.can_const_prop = can_const_prop; } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 68e50070e56..45bd98f39d2 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -247,6 +247,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn remove_const(ecx: &mut InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, local: Local) { ecx.frame_mut().locals[local].value = LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); + ecx.machine.written_only_inside_own_block_locals.remove(&local); } fn lint_root(&self, source_info: SourceInfo) -> Option<HirId> { @@ -484,7 +485,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Some(()) } - fn ensure_not_propagated(&mut self, local: Local) { + fn ensure_not_propagated(&self, local: Local) { if cfg!(debug_assertions) { assert!( self.get_const(local.into()).is_none() @@ -691,17 +692,31 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { // We remove all Locals which are restricted in propagation to their containing blocks and // which were modified in the current block. // Take it out of the ecx so we can get a mutable reference to the ecx for `remove_const`. - let can_const_prop = std::mem::take(&mut self.ecx.machine.can_const_prop); - for (local, &mode) in can_const_prop.iter_enumerated() { - match mode { - ConstPropMode::FullConstProp => {} - ConstPropMode::NoPropagation => self.ensure_not_propagated(local), - ConstPropMode::OnlyInsideOwnBlock => { - Self::remove_const(&mut self.ecx, local); - self.ensure_not_propagated(local); + let mut written_only_inside_own_block_locals = + std::mem::take(&mut self.ecx.machine.written_only_inside_own_block_locals); + + // This loop can get very hot for some bodies: it check each local in each bb. + // To avoid this quadratic behaviour, we only clear the locals that were modified inside + // the current block. + for local in written_only_inside_own_block_locals.drain() { + debug_assert_eq!( + self.ecx.machine.can_const_prop[local], + ConstPropMode::OnlyInsideOwnBlock + ); + Self::remove_const(&mut self.ecx, local); + } + self.ecx.machine.written_only_inside_own_block_locals = + written_only_inside_own_block_locals; + + if cfg!(debug_assertions) { + for (local, &mode) in self.ecx.machine.can_const_prop.iter_enumerated() { + match mode { + ConstPropMode::FullConstProp => {} + ConstPropMode::NoPropagation | ConstPropMode::OnlyInsideOwnBlock => { + self.ensure_not_propagated(local); + } } } } - self.ecx.machine.can_const_prop = can_const_prop; } } diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 49ded10ba1f..a7218a4f250 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -8,6 +8,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_hir::def::DefKind; use rustc_middle::mir::visit::{MutVisitor, Visitor}; use rustc_middle::mir::*; +use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{Map, State, TrackElem, ValueAnalysis, ValueOrPlace}; use rustc_mir_dataflow::{lattice::FlatSet, Analysis, ResultsVisitor, SwitchIntEdgeEffects}; @@ -548,7 +549,7 @@ impl<'mir, 'tcx> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachi unimplemented!() } - fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool { + fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool { unimplemented!() } fn alignment_check_failed( diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index 89ca04a1582..e5c3fa5646a 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -5,7 +5,7 @@ //! purposes on a best-effort basis. We compute them here and store them into the crate metadata so //! dependent crates can use them. -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::LocalDefId; use rustc_index::bit_set::BitSet; use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{Body, Local, Location, Operand, Terminator, TerminatorKind, RETURN_PLACE}; @@ -149,7 +149,10 @@ fn type_will_always_be_passed_directly(ty: Ty<'_>) -> bool { /// body of the function instead of just the signature. These can be useful for optimization /// purposes on a best-effort basis. We compute them here and store them into the crate metadata so /// dependent crates can use them. -pub fn deduced_param_attrs<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx [DeducedParamAttrs] { +pub fn deduced_param_attrs<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: LocalDefId, +) -> &'tcx [DeducedParamAttrs] { // This computation is unfortunately rather expensive, so don't do it unless we're optimizing. // Also skip it in incremental mode. if tcx.sess.opts.optimize == OptLevel::No || tcx.sess.opts.incremental.is_some() { @@ -182,10 +185,6 @@ pub fn deduced_param_attrs<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx [Ded return &[]; } - // Deduced attributes for other crates should be read from the metadata instead of via this - // function. - debug_assert!(def_id.is_local()); - // Grab the optimized MIR. Analyze it to determine which arguments have been mutated. let body: &Body<'tcx> = tcx.optimized_mir(def_id); let mut deduce_read_only = DeduceReadOnly::new(body.arg_count); diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs index e6546911a2d..c9b24adba0c 100644 --- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs +++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs @@ -1,5 +1,6 @@ -use rustc_hir::def_id::{CrateNum, LocalDefId, LOCAL_CRATE}; +use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE}; use rustc_middle::mir::*; +use rustc_middle::query::LocalCrate; use rustc_middle::ty::layout; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt}; @@ -121,9 +122,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool { tainted } -fn required_panic_strategy(tcx: TyCtxt<'_>, cnum: CrateNum) -> Option<PanicStrategy> { - assert_eq!(cnum, LOCAL_CRATE); - +fn required_panic_strategy(tcx: TyCtxt<'_>, _: LocalCrate) -> Option<PanicStrategy> { if tcx.is_panic_runtime(LOCAL_CRATE) { return Some(tcx.sess.panic_strategy()); } diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index e6875fad306..8a6360114dc 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -70,7 +70,7 @@ use rustc_mir_dataflow::impls::{ }; use rustc_mir_dataflow::storage::always_storage_live_locals; use rustc_mir_dataflow::{self, Analysis}; -use rustc_span::def_id::DefId; +use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::symbol::sym; use rustc_span::Span; use rustc_target::abi::VariantIdx; @@ -1386,10 +1386,9 @@ fn create_cases<'tcx>( #[instrument(level = "debug", skip(tcx), ret)] pub(crate) fn mir_generator_witnesses<'tcx>( tcx: TyCtxt<'tcx>, - def_id: DefId, + def_id: LocalDefId, ) -> GeneratorLayout<'tcx> { assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir); - let def_id = def_id.expect_local(); let (body, _) = tcx.mir_promoted(ty::WithOptConstParam::unknown(def_id)); let body = body.borrow(); diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 50c3023b02b..3a515fe8323 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -112,7 +112,6 @@ pub fn provide(providers: &mut Providers) { mir_keys, mir_const, mir_const_qualif: |tcx, def_id| { - let def_id = def_id.expect_local(); if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { tcx.mir_const_qualif_const_arg(def) } else { @@ -133,7 +132,6 @@ pub fn provide(providers: &mut Providers) { mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable, mir_inliner_callees: inline::cycle::mir_inliner_callees, promoted_mir: |tcx, def_id| { - let def_id = def_id.expect_local(); if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { tcx.promoted_mir_of_const_arg(def) } else { @@ -206,8 +204,7 @@ fn remap_mir_for_const_eval_select<'tcx>( body } -fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - let def_id = def_id.expect_local(); +fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { tcx.mir_keys(()).contains(&def_id) } @@ -350,12 +347,11 @@ fn mir_promoted( } /// Compute the MIR that is used during CTFE (and thus has no optimizations run on it) -fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { - let did = def_id.expect_local(); - if let Some(def) = ty::WithOptConstParam::try_lookup(did, tcx) { +fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> { + if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { tcx.mir_for_ctfe_of_const_arg(def) } else { - tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(did))) + tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(def_id))) } } @@ -599,8 +595,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } /// Optimize the MIR and prepare it for codegen. -fn optimized_mir(tcx: TyCtxt<'_>, did: DefId) -> &Body<'_> { - let did = did.expect_local(); +fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> { assert_eq!(ty::WithOptConstParam::try_lookup(did, tcx), None); tcx.arena.alloc(inner_optimized_mir(tcx, did)) } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 5d7382305ae..6a7ceb8fef7 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -6,6 +6,7 @@ use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; use rustc_span::Span; +use rustc_target::abi::VariantIdx; pub struct LowerIntrinsics; @@ -191,6 +192,61 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Goto { target }; } } + sym::option_payload_ptr => { + if let (Some(target), Some(arg)) = (*target, args[0].place()) { + let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) = + destination.ty(local_decls, tcx).ty.kind() + else { bug!(); }; + + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + *destination, + Rvalue::AddressOf( + Mutability::Not, + arg.project_deeper( + &[ + PlaceElem::Deref, + PlaceElem::Downcast( + Some(sym::Some), + VariantIdx::from_u32(1), + ), + PlaceElem::Field(Field::from_u32(0), *dest_ty), + ], + tcx, + ), + ), + ))), + }); + terminator.kind = TerminatorKind::Goto { target }; + } + } + sym::transmute => { + let dst_ty = destination.ty(local_decls, tcx).ty; + let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else { + span_bug!( + terminator.source_info.span, + "Wrong number of arguments for transmute intrinsic", + ); + }; + + // Always emit the cast, even if we transmute to an uninhabited type, + // because that lets CTFE and codegen generate better error messages + // when such a transmute actually ends up reachable. + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + *destination, + Rvalue::Cast(CastKind::Transmute, arg, dst_ty), + ))), + }); + + if let Some(target) = *target { + terminator.kind = TerminatorKind::Goto { target }; + } else { + terminator.kind = TerminatorKind::Unreachable; + } + } _ if intrinsic_name.as_str().starts_with("simd_shuffle") => { validate_simd_shuffle(tcx, args, terminator.source_info.span); } |