diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/check_const_item_mutation.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/mappings.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/spans.rs | 107 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs | 18 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/gvn.rs | 202 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/lib.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/ssa.rs | 38 |
7 files changed, 157 insertions, 223 deletions
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index ceea72c6755..375db17fb73 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -53,9 +53,13 @@ impl<'tcx> ConstMutationChecker<'_, 'tcx> { // // #[const_mutation_allowed] // pub const LOG: Log = Log { msg: "" }; - match self.tcx.calculate_dtor(def_id, |_, _| Ok(())) { - Some(_) => None, - None => Some(def_id), + // FIXME: this should not be checking for `Drop` impls, + // but whether it or any field has a Drop impl (`needs_drop`) + // as fields' Drop impls may make this observable, too. + match self.tcx.type_of(def_id).skip_binder().ty_adt_def().map(|adt| adt.has_dtor(self.tcx)) + { + Some(true) => None, + Some(false) | None => Some(def_id), } } diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs index d83c0d40a7e..73bd2d0705e 100644 --- a/compiler/rustc_mir_transform/src/coverage/mappings.rs +++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs @@ -96,7 +96,7 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( } } else { // Extract coverage spans from MIR statements/terminators as normal. - extract_refined_covspans(mir_body, hir_info, graph, &mut code_mappings); + extract_refined_covspans(tcx, mir_body, hir_info, graph, &mut code_mappings); } branch_pairs.extend(extract_branch_pairs(mir_body, hir_info, graph)); diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 8befe9c5d8d..f57a158e3e4 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,7 +1,9 @@ use std::collections::VecDeque; +use std::iter; use rustc_data_structures::fx::FxHashSet; use rustc_middle::mir; +use rustc_middle::ty::TyCtxt; use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span}; use tracing::{debug, debug_span, instrument}; @@ -11,8 +13,9 @@ use crate::coverage::{ExtractedHirInfo, mappings, unexpand}; mod from_mir; -pub(super) fn extract_refined_covspans( - mir_body: &mir::Body<'_>, +pub(super) fn extract_refined_covspans<'tcx>( + tcx: TyCtxt<'tcx>, + mir_body: &mir::Body<'tcx>, hir_info: &ExtractedHirInfo, graph: &CoverageGraph, code_mappings: &mut impl Extend<mappings::CodeMapping>, @@ -50,7 +53,7 @@ pub(super) fn extract_refined_covspans( // First, perform the passes that need macro information. covspans.sort_by(|a, b| graph.cmp_in_dominator_order(a.bcb, b.bcb)); remove_unwanted_expansion_spans(&mut covspans); - split_visible_macro_spans(&mut covspans); + shrink_visible_macro_spans(tcx, &mut covspans); // We no longer need the extra information in `SpanFromMir`, so convert to `Covspan`. let mut covspans = covspans.into_iter().map(SpanFromMir::into_covspan).collect::<Vec<_>>(); @@ -83,9 +86,7 @@ pub(super) fn extract_refined_covspans( // Split the covspans into separate buckets that don't overlap any holes. let buckets = divide_spans_into_buckets(covspans, &holes); - for mut covspans in buckets { - // Make sure each individual bucket is internally sorted. - covspans.sort_by(compare_covspans); + for covspans in buckets { let _span = debug_span!("processing bucket", ?covspans).entered(); let mut covspans = remove_unwanted_overlapping_spans(covspans); @@ -129,82 +130,50 @@ fn remove_unwanted_expansion_spans(covspans: &mut Vec<SpanFromMir>) { } /// When a span corresponds to a macro invocation that is visible from the -/// function body, split it into two parts. The first part covers just the -/// macro name plus `!`, and the second part covers the rest of the macro -/// invocation. This seems to give better results for code that uses macros. -fn split_visible_macro_spans(covspans: &mut Vec<SpanFromMir>) { - let mut extra_spans = vec![]; - - covspans.retain(|covspan| { - let Some(ExpnKind::Macro(MacroKind::Bang, visible_macro)) = covspan.expn_kind else { - return true; - }; - - let split_len = visible_macro.as_str().len() as u32 + 1; - let (before, after) = covspan.span.split_at(split_len); - if !covspan.span.contains(before) || !covspan.span.contains(after) { - // Something is unexpectedly wrong with the split point. - // The debug assertion in `split_at` will have already caught this, - // but in release builds it's safer to do nothing and maybe get a - // bug report for unexpected coverage, rather than risk an ICE. - return true; +/// function body, truncate it to just the macro name plus `!`. +/// This seems to give better results for code that uses macros. +fn shrink_visible_macro_spans(tcx: TyCtxt<'_>, covspans: &mut Vec<SpanFromMir>) { + let source_map = tcx.sess.source_map(); + + for covspan in covspans { + if matches!(covspan.expn_kind, Some(ExpnKind::Macro(MacroKind::Bang, _))) { + covspan.span = source_map.span_through_char(covspan.span, '!'); } - - extra_spans.push(SpanFromMir::new(before, covspan.expn_kind.clone(), covspan.bcb)); - extra_spans.push(SpanFromMir::new(after, covspan.expn_kind.clone(), covspan.bcb)); - false // Discard the original covspan that we just split. - }); - - // The newly-split spans are added at the end, so any previous sorting - // is not preserved. - covspans.extend(extra_spans); + } } /// Uses the holes to divide the given covspans into buckets, such that: -/// - No span in any hole overlaps a bucket (truncating the spans if necessary). +/// - No span in any hole overlaps a bucket (discarding spans if necessary). /// - The spans in each bucket are strictly after all spans in previous buckets, /// and strictly before all spans in subsequent buckets. /// -/// The resulting buckets are sorted relative to each other, but might not be -/// internally sorted. +/// The lists of covspans and holes must be sorted. +/// The resulting buckets are sorted relative to each other, and each bucket's +/// contents are sorted. #[instrument(level = "debug")] fn divide_spans_into_buckets(input_covspans: Vec<Covspan>, holes: &[Hole]) -> Vec<Vec<Covspan>> { debug_assert!(input_covspans.is_sorted_by(|a, b| compare_spans(a.span, b.span).is_le())); debug_assert!(holes.is_sorted_by(|a, b| compare_spans(a.span, b.span).is_le())); - // Now we're ready to start carving holes out of the initial coverage spans, - // and grouping them in buckets separated by the holes. + // Now we're ready to start grouping spans into buckets separated by holes. let mut input_covspans = VecDeque::from(input_covspans); - let mut fragments = vec![]; // For each hole: // - Identify the spans that are entirely or partly before the hole. - // - Put those spans in a corresponding bucket, truncated to the start of the hole. - // - If one of those spans also extends after the hole, put the rest of it - // in a "fragments" vector that is processed by the next hole. + // - Discard any that overlap with the hole. + // - Add the remaining identified spans to the corresponding bucket. let mut buckets = (0..holes.len()).map(|_| vec![]).collect::<Vec<_>>(); for (hole, bucket) in holes.iter().zip(&mut buckets) { - let fragments_from_prev = std::mem::take(&mut fragments); - - // Only inspect spans that precede or overlap this hole, - // leaving the rest to be inspected by later holes. - // (This relies on the spans and holes both being sorted.) - let relevant_input_covspans = - drain_front_while(&mut input_covspans, |c| c.span.lo() < hole.span.hi()); - - for covspan in fragments_from_prev.into_iter().chain(relevant_input_covspans) { - let (before, after) = covspan.split_around_hole_span(hole.span); - bucket.extend(before); - fragments.extend(after); - } + bucket.extend( + drain_front_while(&mut input_covspans, |c| c.span.lo() < hole.span.hi()) + .filter(|c| !c.span.overlaps(hole.span)), + ); } - // After finding the spans before each hole, any remaining fragments/spans - // form their own final bucket, after the final hole. + // Any remaining spans form their own final bucket, after the final hole. // (If there were no holes, this will just be all of the initial spans.) - fragments.extend(input_covspans); - buckets.push(fragments); + buckets.push(Vec::from(input_covspans)); buckets } @@ -215,7 +184,7 @@ fn drain_front_while<'a, T>( queue: &'a mut VecDeque<T>, mut pred_fn: impl FnMut(&T) -> bool, ) -> impl Iterator<Item = T> { - std::iter::from_fn(move || if pred_fn(queue.front()?) { queue.pop_front() } else { None }) + iter::from_fn(move || queue.pop_front_if(|x| pred_fn(x))) } /// Takes one of the buckets of (sorted) spans extracted from MIR, and "refines" @@ -258,22 +227,6 @@ struct Covspan { } impl Covspan { - /// Splits this covspan into 0-2 parts: - /// - The part that is strictly before the hole span, if any. - /// - The part that is strictly after the hole span, if any. - fn split_around_hole_span(&self, hole_span: Span) -> (Option<Self>, Option<Self>) { - let before = try { - let span = self.span.trim_end(hole_span)?; - Self { span, ..*self } - }; - let after = try { - let span = self.span.trim_start(hole_span)?; - Self { span, ..*self } - }; - - (before, after) - } - /// If `self` and `other` can be merged (i.e. they have the same BCB), /// mutates `self.span` to also include `other.span` and returns true. /// diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 1faa2171c0b..804cd8ab3f7 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -120,22 +120,20 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { // an `if condition { block }` has a span that includes the executed block, if true, // but for coverage, the code region executed, up to *and* through the SwitchInt, // actually stops before the if's block.) - TerminatorKind::Unreachable // Unreachable blocks are not connected to the MIR CFG + TerminatorKind::Unreachable | TerminatorKind::Assert { .. } | TerminatorKind::Drop { .. } | TerminatorKind::SwitchInt { .. } - // For `FalseEdge`, only the `real` branch is taken, so it is similar to a `Goto`. | TerminatorKind::FalseEdge { .. } | TerminatorKind::Goto { .. } => None, // Call `func` operand can have a more specific span when part of a chain of calls - TerminatorKind::Call { ref func, .. } - | TerminatorKind::TailCall { ref func, .. } => { + TerminatorKind::Call { ref func, .. } | TerminatorKind::TailCall { ref func, .. } => { let mut span = terminator.source_info.span; - if let mir::Operand::Constant(box constant) = func { - if constant.span.lo() > span.lo() { - span = span.with_lo(constant.span.lo()); - } + if let mir::Operand::Constant(constant) = func + && span.contains(constant.span) + { + span = constant.span; } Some(span) } @@ -147,9 +145,7 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { | TerminatorKind::Yield { .. } | TerminatorKind::CoroutineDrop | TerminatorKind::FalseUnwind { .. } - | TerminatorKind::InlineAsm { .. } => { - Some(terminator.source_info.span) - } + | TerminatorKind::InlineAsm { .. } => Some(terminator.source_info.span), } } diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 0a54c780f31..68bc0ffce6b 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -3,14 +3,16 @@ //! MIR may contain repeated and/or redundant computations. The objective of this pass is to detect //! such redundancies and re-use the already-computed result when possible. //! -//! In a first pass, we compute a symbolic representation of values that are assigned to SSA -//! locals. This symbolic representation is defined by the `Value` enum. Each produced instance of -//! `Value` is interned as a `VnIndex`, which allows us to cheaply compute identical values. -//! //! From those assignments, we construct a mapping `VnIndex -> Vec<(Local, Location)>` of available //! values, the locals in which they are stored, and the assignment location. //! -//! In a second pass, we traverse all (non SSA) assignments `x = rvalue` and operands. For each +//! We traverse all assignments `x = rvalue` and operands. +//! +//! For each SSA one, we compute a symbolic representation of values that are assigned to SSA +//! locals. This symbolic representation is defined by the `Value` enum. Each produced instance of +//! `Value` is interned as a `VnIndex`, which allows us to cheaply compute identical values. +//! +//! For each non-SSA //! one, we compute the `VnIndex` of the rvalue. If this `VnIndex` is associated to a constant, we //! replace the rvalue/operand by that constant. Otherwise, if there is an SSA local `y` //! associated to this `VnIndex`, and if its definition location strictly dominates the assignment @@ -91,7 +93,7 @@ use rustc_const_eval::interpret::{ ImmTy, Immediate, InterpCx, MemPlaceMeta, MemoryKind, OpTy, Projectable, Scalar, intern_const_alloc_for_constprop, }; -use rustc_data_structures::fx::FxIndexSet; +use rustc_data_structures::fx::{FxIndexSet, MutableValues}; use rustc_data_structures::graph::dominators::Dominators; use rustc_hir::def::DefKind; use rustc_index::bit_set::DenseBitSet; @@ -107,7 +109,7 @@ use rustc_span::def_id::DefId; use smallvec::SmallVec; use tracing::{debug, instrument, trace}; -use crate::ssa::{AssignedValue, SsaLocals}; +use crate::ssa::SsaLocals; pub(super) struct GVN; @@ -126,31 +128,11 @@ impl<'tcx> crate::MirPass<'tcx> for GVN { let dominators = body.basic_blocks.dominators().clone(); let mut state = VnState::new(tcx, body, typing_env, &ssa, dominators, &body.local_decls); - ssa.for_each_assignment_mut( - body.basic_blocks.as_mut_preserves_cfg(), - |local, value, location| { - let value = match value { - // We do not know anything of this assigned value. - AssignedValue::Arg | AssignedValue::Terminator => None, - // Try to get some insight. - AssignedValue::Rvalue(rvalue) => { - let value = state.simplify_rvalue(rvalue, location); - // FIXME(#112651) `rvalue` may have a subtype to `local`. We can only mark - // `local` as reusable if we have an exact type match. - if state.local_decls[local].ty != rvalue.ty(state.local_decls, tcx) { - return; - } - value - } - }; - // `next_opaque` is `Some`, so `new_opaque` must return `Some`. - let value = value.or_else(|| state.new_opaque()).unwrap(); - state.assign(local, value); - }, - ); - // Stop creating opaques during replacement as it is useless. - state.next_opaque = None; + for local in body.args_iter().filter(|&local| ssa.is_ssa(local)) { + let opaque = state.new_opaque(); + state.assign(local, opaque); + } let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec(); for bb in reverse_postorder { @@ -250,14 +232,14 @@ struct VnState<'body, 'tcx> { locals: IndexVec<Local, Option<VnIndex>>, /// Locals that are assigned that value. // This vector does not hold all the values of `VnIndex` that we create. - // It stops at the largest value created in the first phase of collecting assignments. rev_locals: IndexVec<VnIndex, SmallVec<[Local; 1]>>, values: FxIndexSet<Value<'tcx>>, /// Values evaluated as constants if possible. evaluated: IndexVec<VnIndex, Option<OpTy<'tcx>>>, /// Counter to generate different values. - /// This is an option to stop creating opaques during replacement. - next_opaque: Option<usize>, + next_opaque: usize, + /// Cache the deref values. + derefs: Vec<VnIndex>, /// Cache the value of the `unsized_locals` features, to avoid fetching it repeatedly in a loop. feature_unsized_locals: bool, ssa: &'body SsaLocals, @@ -289,7 +271,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { rev_locals: IndexVec::with_capacity(num_values), values: FxIndexSet::with_capacity_and_hasher(num_values, Default::default()), evaluated: IndexVec::with_capacity(num_values), - next_opaque: Some(1), + next_opaque: 1, + derefs: Vec::new(), feature_unsized_locals: tcx.features().unsized_locals(), ssa, dominators, @@ -310,32 +293,31 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { let evaluated = self.eval_to_const(index); let _index = self.evaluated.push(evaluated); debug_assert_eq!(index, _index); - // No need to push to `rev_locals` if we finished listing assignments. - if self.next_opaque.is_some() { - let _index = self.rev_locals.push(SmallVec::new()); - debug_assert_eq!(index, _index); - } + let _index = self.rev_locals.push(SmallVec::new()); + debug_assert_eq!(index, _index); } index } + fn next_opaque(&mut self) -> usize { + let next_opaque = self.next_opaque; + self.next_opaque += 1; + next_opaque + } + /// Create a new `Value` for which we have no information at all, except that it is distinct /// from all the others. #[instrument(level = "trace", skip(self), ret)] - fn new_opaque(&mut self) -> Option<VnIndex> { - let next_opaque = self.next_opaque.as_mut()?; - let value = Value::Opaque(*next_opaque); - *next_opaque += 1; - Some(self.insert(value)) + fn new_opaque(&mut self) -> VnIndex { + let value = Value::Opaque(self.next_opaque()); + self.insert(value) } /// Create a new `Value::Address` distinct from all the others. #[instrument(level = "trace", skip(self), ret)] - fn new_pointer(&mut self, place: Place<'tcx>, kind: AddressKind) -> Option<VnIndex> { - let next_opaque = self.next_opaque.as_mut()?; - let value = Value::Address { place, kind, provenance: *next_opaque }; - *next_opaque += 1; - Some(self.insert(value)) + fn new_pointer(&mut self, place: Place<'tcx>, kind: AddressKind) -> VnIndex { + let value = Value::Address { place, kind, provenance: self.next_opaque() }; + self.insert(value) } fn get(&self, index: VnIndex) -> &Value<'tcx> { @@ -345,6 +327,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { /// Record that `local` is assigned `value`. `local` must be SSA. #[instrument(level = "trace", skip(self))] fn assign(&mut self, local: Local, value: VnIndex) { + debug_assert!(self.ssa.is_ssa(local)); self.locals[local] = Some(value); // Only register the value if its type is `Sized`, as we will emit copies of it. @@ -355,21 +338,19 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } } - fn insert_constant(&mut self, value: Const<'tcx>) -> Option<VnIndex> { + fn insert_constant(&mut self, value: Const<'tcx>) -> VnIndex { let disambiguator = if value.is_deterministic() { // The constant is deterministic, no need to disambiguate. 0 } else { // Multiple mentions of this constant will yield different values, // so assign a different `disambiguator` to ensure they do not get the same `VnIndex`. - let next_opaque = self.next_opaque.as_mut()?; - let disambiguator = *next_opaque; - *next_opaque += 1; + let disambiguator = self.next_opaque(); // `disambiguator: 0` means deterministic. debug_assert_ne!(disambiguator, 0); disambiguator }; - Some(self.insert(Value::Constant { value, disambiguator })) + self.insert(Value::Constant { value, disambiguator }) } fn insert_bool(&mut self, flag: bool) -> VnIndex { @@ -390,6 +371,19 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { self.insert(Value::Aggregate(AggregateTy::Tuple, VariantIdx::ZERO, values)) } + fn insert_deref(&mut self, value: VnIndex) -> VnIndex { + let value = self.insert(Value::Projection(value, ProjectionElem::Deref)); + self.derefs.push(value); + value + } + + fn invalidate_derefs(&mut self) { + for deref in std::mem::take(&mut self.derefs) { + let opaque = self.next_opaque(); + *self.values.get_index_mut2(deref.index()).unwrap() = Value::Opaque(opaque); + } + } + #[instrument(level = "trace", skip(self), ret)] fn eval_to_const(&mut self, value: VnIndex) -> Option<OpTy<'tcx>> { use Value::*; @@ -648,15 +642,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { let proj = match proj { ProjectionElem::Deref => { let ty = place.ty(self.local_decls, self.tcx).ty; - // unsound: https://github.com/rust-lang/rust/issues/130853 - if self.tcx.sess.opts.unstable_opts.unsound_mir_opts - && let Some(Mutability::Not) = ty.ref_mutability() + if let Some(Mutability::Not) = ty.ref_mutability() && let Some(pointee_ty) = ty.builtin_deref(true) && pointee_ty.is_freeze(self.tcx, self.typing_env()) { // An immutable borrow `_x` always points to the same value for the // lifetime of the borrow, so we can merge all instances of `*_x`. - ProjectionElem::Deref + return Some(self.insert_deref(value)); } else { return None; } @@ -830,7 +822,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { location: Location, ) -> Option<VnIndex> { match *operand { - Operand::Constant(ref constant) => self.insert_constant(constant.const_), + Operand::Constant(ref constant) => Some(self.insert_constant(constant.const_)), Operand::Copy(ref mut place) | Operand::Move(ref mut place) => { let value = self.simplify_place_value(place, location)?; if let Some(const_) = self.try_as_constant(value) { @@ -866,11 +858,11 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { Rvalue::Aggregate(..) => return self.simplify_aggregate(rvalue, location), Rvalue::Ref(_, borrow_kind, ref mut place) => { self.simplify_place_projection(place, location); - return self.new_pointer(*place, AddressKind::Ref(borrow_kind)); + return Some(self.new_pointer(*place, AddressKind::Ref(borrow_kind))); } Rvalue::RawPtr(mutbl, ref mut place) => { self.simplify_place_projection(place, location); - return self.new_pointer(*place, AddressKind::Address(mutbl)); + return Some(self.new_pointer(*place, AddressKind::Address(mutbl))); } Rvalue::WrapUnsafeBinder(ref mut op, ty) => { let value = self.simplify_operand(op, location)?; @@ -1034,7 +1026,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { if is_zst { let ty = rvalue.ty(self.local_decls, tcx); - return self.insert_constant(Const::zero_sized(ty)); + return Some(self.insert_constant(Const::zero_sized(ty))); } } @@ -1063,11 +1055,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } }; - let fields: Option<Vec<_>> = field_ops + let mut fields: Vec<_> = field_ops .iter_mut() - .map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque())) + .map(|op| self.simplify_operand(op, location).unwrap_or_else(|| self.new_opaque())) .collect(); - let mut fields = fields?; if let AggregateTy::RawPtr { data_pointer_ty, output_pointer_ty } = &mut ty { let mut was_updated = false; @@ -1107,9 +1098,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } } - // unsound: https://github.com/rust-lang/rust/issues/132353 - if tcx.sess.opts.unstable_opts.unsound_mir_opts - && let AggregateTy::Def(_, _) = ty + if let AggregateTy::Def(_, _) = ty && let Some(value) = self.simplify_aggregate_to_copy(rvalue, location, &fields, variant_index) { @@ -1195,7 +1184,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { ) if let ty::Slice(..) = to.builtin_deref(true).unwrap().kind() && let ty::Array(_, len) = from.builtin_deref(true).unwrap().kind() => { - return self.insert_constant(Const::Ty(self.tcx.types.usize, *len)); + return Some(self.insert_constant(Const::Ty(self.tcx.types.usize, *len))); } _ => Value::UnaryOp(op, arg_index), }; @@ -1391,7 +1380,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { if let CastKind::PointerCoercion(ReifyFnPointer | ClosureFnPointer(_), _) = kind { // Each reification of a generic fn may get a different pointer. // Do not try to merge them. - return self.new_opaque(); + return Some(self.new_opaque()); } let mut was_ever_updated = false; @@ -1507,7 +1496,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { // Trivial case: we are fetching a statically known length. let place_ty = place.ty(self.local_decls, self.tcx).ty; if let ty::Array(_, len) = place_ty.kind() { - return self.insert_constant(Const::Ty(self.tcx.types.usize, *len)); + return Some(self.insert_constant(Const::Ty(self.tcx.types.usize, *len))); } let mut inner = self.simplify_place_value(place, location)?; @@ -1529,7 +1518,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { && let Some(to) = to.builtin_deref(true) && let ty::Slice(..) = to.kind() { - return self.insert_constant(Const::Ty(self.tcx.types.usize, *len)); + return Some(self.insert_constant(Const::Ty(self.tcx.types.usize, *len))); } // Fallback: a symbolic `Len`. @@ -1739,41 +1728,70 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> { self.tcx } - fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, location: Location) { + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { self.simplify_place_projection(place, location); + if context.is_mutating_use() && !place.projection.is_empty() { + // Non-local mutation maybe invalidate deref. + self.invalidate_derefs(); + } + self.super_place(place, context, location); } fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { self.simplify_operand(operand, location); + self.super_operand(operand, location); } fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) { if let StatementKind::Assign(box (ref mut lhs, ref mut rvalue)) = stmt.kind { self.simplify_place_projection(lhs, location); - // Do not try to simplify a constant, it's already in canonical shape. - if matches!(rvalue, Rvalue::Use(Operand::Constant(_))) { - return; + let value = self.simplify_rvalue(rvalue, location); + let value = if let Some(local) = lhs.as_local() + && self.ssa.is_ssa(local) + // FIXME(#112651) `rvalue` may have a subtype to `local`. We can only mark + // `local` as reusable if we have an exact type match. + && self.local_decls[local].ty == rvalue.ty(self.local_decls, self.tcx) + { + let value = value.unwrap_or_else(|| self.new_opaque()); + self.assign(local, value); + Some(value) + } else { + value + }; + if let Some(value) = value { + if let Some(const_) = self.try_as_constant(value) { + *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); + } else if let Some(local) = self.try_as_local(value, location) + && *rvalue != Rvalue::Use(Operand::Move(local.into())) + { + *rvalue = Rvalue::Use(Operand::Copy(local.into())); + self.reused_locals.insert(local); + } } + } + self.super_statement(stmt, location); + } - let value = lhs - .as_local() - .and_then(|local| self.locals[local]) - .or_else(|| self.simplify_rvalue(rvalue, location)); - let Some(value) = value else { return }; - - if let Some(const_) = self.try_as_constant(value) { - *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); - } else if let Some(local) = self.try_as_local(value, location) - && *rvalue != Rvalue::Use(Operand::Move(local.into())) + fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { + if let Terminator { kind: TerminatorKind::Call { destination, .. }, .. } = terminator { + if let Some(local) = destination.as_local() + && self.ssa.is_ssa(local) { - *rvalue = Rvalue::Use(Operand::Copy(local.into())); - self.reused_locals.insert(local); + let opaque = self.new_opaque(); + self.assign(local, opaque); } - - return; } - self.super_statement(stmt, location); + // Function calls and ASM may invalidate (nested) derefs. We must handle them carefully. + // Currently, only preserving derefs for trivial terminators like SwitchInt and Goto. + let safe_to_preserve_derefs = matches!( + terminator.kind, + TerminatorKind::SwitchInt { .. } | TerminatorKind::Goto { .. } + ); + if !safe_to_preserve_derefs { + self.invalidate_derefs(); + } + self.super_terminator(terminator, location); } } diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 205d388f4fb..6429d3f67ec 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -12,6 +12,7 @@ #![feature(map_try_insert)] #![feature(never_type)] #![feature(try_blocks)] +#![feature(vec_deque_pop_if)] #![feature(yeet_expr)] // tidy-alphabetical-end @@ -528,7 +529,7 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & | DefKind::Static { .. } | DefKind::Const | DefKind::AssocConst => { - if let Err(guar) = tcx.check_well_formed(root.expect_local()) { + if let Err(guar) = tcx.ensure_ok().check_well_formed(root.expect_local()) { body.tainted_by_errors = Some(guar); } } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 3d512fb064e..edd0cabca49 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -32,12 +32,6 @@ pub(super) struct SsaLocals { borrowed_locals: DenseBitSet<Local>, } -pub(super) enum AssignedValue<'a, 'tcx> { - Arg, - Rvalue(&'a mut Rvalue<'tcx>), - Terminator, -} - impl SsaLocals { pub(super) fn new<'tcx>( tcx: TyCtxt<'tcx>, @@ -152,38 +146,6 @@ impl SsaLocals { }) } - pub(super) fn for_each_assignment_mut<'tcx>( - &self, - basic_blocks: &mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>, - mut f: impl FnMut(Local, AssignedValue<'_, 'tcx>, Location), - ) { - for &local in &self.assignment_order { - match self.assignments[local] { - Set1::One(DefLocation::Argument) => f( - local, - AssignedValue::Arg, - Location { block: START_BLOCK, statement_index: 0 }, - ), - Set1::One(DefLocation::Assignment(loc)) => { - let bb = &mut basic_blocks[loc.block]; - // `loc` must point to a direct assignment to `local`. - let stmt = &mut bb.statements[loc.statement_index]; - let StatementKind::Assign(box (target, ref mut rvalue)) = stmt.kind else { - bug!() - }; - assert_eq!(target.as_local(), Some(local)); - f(local, AssignedValue::Rvalue(rvalue), loc) - } - Set1::One(DefLocation::CallReturn { call, .. }) => { - let bb = &mut basic_blocks[call]; - let loc = Location { block: call, statement_index: bb.statements.len() }; - f(local, AssignedValue::Terminator, loc) - } - _ => {} - } - } - } - /// Compute the equivalence classes for locals, based on copy statements. /// /// The returned vector maps each local to the one it copies. In the following case: |