diff options
Diffstat (limited to 'compiler/rustc_mir_transform')
32 files changed, 429 insertions, 220 deletions
diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl index 8c85cb5f76d..b13429d121d 100644 --- a/compiler/rustc_mir_transform/messages.ftl +++ b/compiler/rustc_mir_transform/messages.ftl @@ -1,3 +1,8 @@ +mir_transform_arithmetic_overflow = this arithmetic operation will overflow +mir_transform_call_to_unsafe_label = call to unsafe function +mir_transform_call_to_unsafe_note = consult the function's documentation for information on how to avoid undefined behavior +mir_transform_const_defined_here = `const` item defined here + mir_transform_const_modify = attempting to modify a `const` item .note = each usage of a `const` item creates a new temporary; the original `const` item will not be modified @@ -6,50 +11,10 @@ mir_transform_const_mut_borrow = taking a mutable reference to a `const` item .note2 = the mutable reference will refer to this temporary, not the original `const` item .note3 = mutable reference created due to call to this method -mir_transform_const_defined_here = `const` item defined here - -mir_transform_unaligned_packed_ref = reference to packed field is unaligned - .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses - .note_ub = creating a misaligned reference is undefined behavior (even if that reference is never dereferenced) - .help = copy the field contents to a local variable, or replace the reference with a raw pointer and use `read_unaligned`/`write_unaligned` (loads and stores via `*p` must be properly aligned even when using raw pointers) - -mir_transform_unused_unsafe = unnecessary `unsafe` block - .label = because it's nested under this `unsafe` block - -mir_transform_requires_unsafe = {$details} is unsafe and requires unsafe {$op_in_unsafe_fn_allowed -> - [true] function or block - *[false] block - } - .not_inherited = items do not inherit unsafety from separate enclosing items - -mir_transform_call_to_unsafe_label = call to unsafe function -mir_transform_call_to_unsafe_note = consult the function's documentation for information on how to avoid undefined behavior -mir_transform_use_of_asm_label = use of inline assembly -mir_transform_use_of_asm_note = inline assembly is entirely unchecked and can cause undefined behavior -mir_transform_initializing_valid_range_label = initializing type with `rustc_layout_scalar_valid_range` attr -mir_transform_initializing_valid_range_note = initializing a layout restricted type's field with a value outside the valid range is undefined behavior mir_transform_const_ptr2int_label = cast of pointer to int mir_transform_const_ptr2int_note = casting pointers to integers in constants -mir_transform_use_of_static_mut_label = use of mutable static -mir_transform_use_of_static_mut_note = mutable statics can be mutated by multiple threads: aliasing violations or data races will cause undefined behavior -mir_transform_use_of_extern_static_label = use of extern static -mir_transform_use_of_extern_static_note = extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior mir_transform_deref_ptr_label = dereference of raw pointer mir_transform_deref_ptr_note = raw pointers may be null, dangling or unaligned; they can violate aliasing rules and cause data races: all of these are undefined behavior -mir_transform_union_access_label = access to union field -mir_transform_union_access_note = the field may not be properly initialized: using uninitialized data will cause undefined behavior -mir_transform_mutation_layout_constrained_label = mutation of layout constrained field -mir_transform_mutation_layout_constrained_note = mutating layout constrained fields cannot statically be checked for valid values -mir_transform_mutation_layout_constrained_borrow_label = borrow of layout constrained field with interior mutability -mir_transform_mutation_layout_constrained_borrow_note = references to fields of layout constrained fields lose the constraints. Coupled with interior mutability, the field can be changed to invalid values -mir_transform_target_feature_call_label = call to function with `#[target_feature]` -mir_transform_target_feature_call_note = can only be called if the required target features are available - -mir_transform_unsafe_op_in_unsafe_fn = {$details} is unsafe and requires unsafe block (error E0133) - -mir_transform_arithmetic_overflow = this arithmetic operation will overflow -mir_transform_operation_will_panic = this operation will panic at runtime - mir_transform_ffi_unwind_call = call to {$foreign -> [true] foreign function *[false] function pointer @@ -58,9 +23,45 @@ mir_transform_ffi_unwind_call = call to {$foreign -> mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer .suggestion = cast `{$ident}` to obtain a function pointer +mir_transform_initializing_valid_range_label = initializing type with `rustc_layout_scalar_valid_range` attr +mir_transform_initializing_valid_range_note = initializing a layout restricted type's field with a value outside the valid range is undefined behavior mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be .label = the value is held across this suspend point .note = {$reason} .help = consider using a block (`{"{ ... }"}`) to shrink the value's scope, ending before the suspend point +mir_transform_mutation_layout_constrained_borrow_label = borrow of layout constrained field with interior mutability +mir_transform_mutation_layout_constrained_borrow_note = references to fields of layout constrained fields lose the constraints. Coupled with interior mutability, the field can be changed to invalid values +mir_transform_mutation_layout_constrained_label = mutation of layout constrained field +mir_transform_mutation_layout_constrained_note = mutating layout constrained fields cannot statically be checked for valid values +mir_transform_operation_will_panic = this operation will panic at runtime + +mir_transform_requires_unsafe = {$details} is unsafe and requires unsafe {$op_in_unsafe_fn_allowed -> + [true] function or block + *[false] block + } + .not_inherited = items do not inherit unsafety from separate enclosing items + mir_transform_simd_shuffle_last_const = last argument of `simd_shuffle` is required to be a `const` item + +mir_transform_target_feature_call_label = call to function with `#[target_feature]` +mir_transform_target_feature_call_note = can only be called if the required target features are available + +mir_transform_unaligned_packed_ref = reference to packed field is unaligned + .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses + .note_ub = creating a misaligned reference is undefined behavior (even if that reference is never dereferenced) + .help = copy the field contents to a local variable, or replace the reference with a raw pointer and use `read_unaligned`/`write_unaligned` (loads and stores via `*p` must be properly aligned even when using raw pointers) + +mir_transform_union_access_label = access to union field +mir_transform_union_access_note = the field may not be properly initialized: using uninitialized data will cause undefined behavior +mir_transform_unsafe_op_in_unsafe_fn = {$details} is unsafe and requires unsafe block (error E0133) + +mir_transform_unused_unsafe = unnecessary `unsafe` block + .label = because it's nested under this `unsafe` block + +mir_transform_use_of_asm_label = use of inline assembly +mir_transform_use_of_asm_note = inline assembly is entirely unchecked and can cause undefined behavior +mir_transform_use_of_extern_static_label = use of extern static +mir_transform_use_of_extern_static_note = extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior +mir_transform_use_of_static_mut_label = use of mutable static +mir_transform_use_of_static_mut_note = mutable statics can be mutated by multiple threads: aliasing violations or data races will cause undefined behavior diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index b29ffcc70f9..ef2a0c790e9 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -80,7 +80,7 @@ fn add_move_for_packed_drop<'tcx>( is_cleanup: bool, ) { debug!("add_move_for_packed_drop({:?} @ {:?})", terminator, loc); - let TerminatorKind::Drop { ref place, target, unwind } = terminator.kind else { + let TerminatorKind::Drop { ref place, target, unwind, replace } = terminator.kind else { unreachable!(); }; @@ -98,6 +98,11 @@ fn add_move_for_packed_drop<'tcx>( patch.add_assign(loc, Place::from(temp), Rvalue::Use(Operand::Move(*place))); patch.patch_terminator( loc.block, - TerminatorKind::Drop { place: Place::from(temp), target: storage_dead_block, unwind }, + TerminatorKind::Drop { + place: Place::from(temp), + target: storage_dead_block, + unwind, + replace, + }, ); } diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index d60184e0ebe..1fe8ea07892 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -75,6 +75,14 @@ struct PointerFinder<'tcx, 'a> { } impl<'tcx, 'a> Visitor<'tcx> for PointerFinder<'tcx, 'a> { + fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { + if let Rvalue::AddressOf(..) = rvalue { + // Ignore dereferences inside of an AddressOf + return; + } + self.super_rvalue(rvalue, location); + } + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { if let PlaceContext::NonUse(_) = context { return; diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 069514d8a3b..2f851cd1eb5 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -421,10 +421,8 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> { intravisit::walk_block(self, block); } - fn visit_anon_const(&mut self, c: &'tcx hir::AnonConst) { - if matches!(self.tcx.def_kind(c.def_id), DefKind::InlineConst) { - self.visit_body(self.tcx.hir().body(c.body)) - } + fn visit_inline_const(&mut self, c: &'tcx hir::ConstBlock) { + self.visit_body(self.tcx.hir().body(c.body)) } fn visit_fn( diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index da101ca7ad2..024bea62098 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -28,7 +28,7 @@ pub struct ConstGoto; impl<'tcx> MirPass<'tcx> for ConstGoto { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 4 + sess.mir_opt_level() >= 2 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index a5d18fff89b..1d43dbda0aa 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -4,6 +4,7 @@ use either::Right; use rustc_const_eval::const_eval::CheckAlignment; +use rustc_const_eval::ReportErrorExt; use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; @@ -37,6 +38,7 @@ macro_rules! throw_machine_stop_str { ($($tt:tt)*) => {{ // We make a new local type for it. The type itself does not carry any information, // but its vtable (for the `MachineStopType` trait) does. + #[derive(Debug)] struct Zst; // Printing this type shows the desired string. impl std::fmt::Display for Zst { @@ -44,7 +46,17 @@ macro_rules! throw_machine_stop_str { write!(f, $($tt)*) } } - impl rustc_middle::mir::interpret::MachineStopType for Zst {} + + impl rustc_middle::mir::interpret::MachineStopType for Zst { + fn diagnostic_message(&self) -> rustc_errors::DiagnosticMessage { + self.to_string().into() + } + + fn add_args( + self: Box<Self>, + _: &mut dyn FnMut(std::borrow::Cow<'static, str>, rustc_errors::DiagnosticArgValue<'static>), + ) {} + } throw_machine_stop!(Zst) }}; } @@ -103,7 +115,14 @@ impl<'tcx> MirPass<'tcx> for ConstProp { // That would require a uniform one-def no-mutation analysis // and RPO (or recursing when needing the value of a local). let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); - optimization_finder.visit_body(body); + + // Traverse the body in reverse post-order, to ensure that `FullConstProp` locals are + // assigned before being read. + let postorder = body.basic_blocks.postorder().to_vec(); + for bb in postorder.into_iter().rev() { + let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; + optimization_finder.visit_basic_block_data(bb, data); + } trace!("ConstProp done for {:?}", def_id); } @@ -367,7 +386,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { op } Err(e) => { - trace!("get_const failed: {}", e); + trace!("get_const failed: {:?}", e.into_kind().debug()); return None; } }; @@ -772,7 +791,6 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // mutation. | NonMutatingUse(NonMutatingUseContext::SharedBorrow) | NonMutatingUse(NonMutatingUseContext::ShallowBorrow) - | NonMutatingUse(NonMutatingUseContext::UniqueBorrow) | NonMutatingUse(NonMutatingUseContext::AddressOf) | MutatingUse(MutatingUseContext::Borrow) | MutatingUse(MutatingUseContext::AddressOf) => { @@ -790,12 +808,6 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { self.tcx } - fn visit_body(&mut self, body: &mut Body<'tcx>) { - for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { - self.visit_basic_block_data(bb, data); - } - } - fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { self.super_operand(operand, location); @@ -886,14 +898,23 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } StatementKind::StorageLive(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); - } - StatementKind::StorageDead(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].value = LocalValue::Dead; + Self::remove_const(&mut self.ecx, local); } + // We do not need to mark dead locals as such. For `FullConstProp` locals, + // this allows to propagate the single assigned value in this case: + // ``` + // let x = SOME_CONST; + // if a { + // f(copy x); + // StorageDead(x); + // } else { + // g(copy x); + // StorageDead(x); + // } + // ``` + // + // This may propagate a constant where the local would be uninit or dead. + // In both cases, this does not matter, as those reads would be UB anyway. _ => {} } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index adb09c509d2..759650fe4db 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -9,6 +9,7 @@ use rustc_const_eval::interpret::Immediate; use rustc_const_eval::interpret::{ self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, }; +use rustc_const_eval::ReportErrorExt; use rustc_hir::def::DefKind; use rustc_hir::HirId; use rustc_index::bit_set::BitSet; @@ -232,7 +233,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { op } Err(e) => { - trace!("get_const failed: {}", e); + trace!("get_const failed: {:?}", e.into_kind().debug()); return None; } }; @@ -272,8 +273,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // dedicated error variants should be introduced instead. assert!( !error.kind().formatted_string(), - "const-prop encountered formatting error: {}", - error + "const-prop encountered formatting error: {error:?}", ); None } @@ -363,7 +363,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let left_size = self.ecx.layout_of(left_ty).ok()?.size; let right_size = r.layout.size; let r_bits = r.to_scalar().to_bits(right_size).ok(); - if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { + if r_bits.is_some_and(|b| b >= left_size.bits() as u128) { debug!("check_binary_op: reporting assert for {:?}", location); let source_info = self.body().source_info(location); let panic = AssertKind::Overflow( diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 319f3a79705..3df459dfa79 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -130,7 +130,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { PlaceContext::NonMutatingUse( NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::UniqueBorrow | NonMutatingUseContext::AddressOf, ) => true, // For debuginfo, merging locals is ok. diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index e554c470646..6a3d42511ac 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -118,7 +118,7 @@ use rustc_middle::mir::spanview::{self, SpanViewable}; use rustc_data_structures::fx::FxHashMap; use rustc_middle::mir::coverage::*; -use rustc_middle::mir::{self, BasicBlock, TerminatorKind}; +use rustc_middle::mir::{self, BasicBlock}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; @@ -639,7 +639,7 @@ pub(super) fn dump_coverage_spanview<'tcx>( let def_id = mir_source.def_id(); let span_viewables = span_viewables(tcx, mir_body, basic_coverage_blocks, &coverage_spans); - let mut file = create_dump_file(tcx, "html", false, pass_name, &0, mir_body) + let mut file = create_dump_file(tcx, "html", false, pass_name, &0i32, mir_body) .expect("Unexpected error creating MIR spanview HTML file"); let crate_name = tcx.crate_name(def_id.krate); let item_name = tcx.def_path(def_id).to_filename_friendly_no_crate(); @@ -740,7 +740,7 @@ pub(super) fn dump_coverage_graphviz<'tcx>( .join("\n ") )); } - let mut file = create_dump_file(tcx, "dot", false, pass_name, &0, mir_body) + let mut file = create_dump_file(tcx, "dot", false, pass_name, &0i32, mir_body) .expect("Unexpected error creating BasicCoverageBlock graphviz DOT file"); graphviz_writer .write_graphviz(tcx, &mut file) @@ -796,7 +796,7 @@ fn bcb_to_string_sections<'tcx>( } let non_term_blocks = bcb_data.basic_blocks[0..len - 1] .iter() - .map(|&bb| format!("{:?}: {}", bb, term_type(&mir_body[bb].terminator().kind))) + .map(|&bb| format!("{:?}: {}", bb, mir_body[bb].terminator().kind.name())) .collect::<Vec<_>>(); if non_term_blocks.len() > 0 { sections.push(non_term_blocks.join("\n")); @@ -804,28 +804,7 @@ fn bcb_to_string_sections<'tcx>( sections.push(format!( "{:?}: {}", bcb_data.basic_blocks.last().unwrap(), - term_type(&bcb_data.terminator(mir_body).kind) + bcb_data.terminator(mir_body).kind.name(), )); sections } - -/// Returns a simple string representation of a `TerminatorKind` variant, independent of any -/// values it might hold. -pub(super) fn term_type(kind: &TerminatorKind<'_>) -> &'static str { - match kind { - TerminatorKind::Goto { .. } => "Goto", - TerminatorKind::SwitchInt { .. } => "SwitchInt", - TerminatorKind::Resume => "Resume", - TerminatorKind::Terminate => "Terminate", - TerminatorKind::Return => "Return", - TerminatorKind::Unreachable => "Unreachable", - TerminatorKind::Drop { .. } => "Drop", - TerminatorKind::Call { .. } => "Call", - TerminatorKind::Assert { .. } => "Assert", - TerminatorKind::Yield { .. } => "Yield", - TerminatorKind::GeneratorDrop => "GeneratorDrop", - TerminatorKind::FalseEdge { .. } => "FalseEdge", - TerminatorKind::FalseUnwind { .. } => "FalseUnwind", - TerminatorKind::InlineAsm { .. } => "InlineAsm", - } -} diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 986d2fd190d..ea1223fbca6 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -9,6 +9,7 @@ use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{self, BasicBlock, BasicBlockData, Terminator, TerminatorKind}; +use std::cmp::Ordering; use std::ops::{Index, IndexMut}; const ID_SEPARATOR: &str = ","; @@ -212,8 +213,12 @@ impl CoverageGraph { } #[inline(always)] - pub fn dominators(&self) -> &Dominators<BasicCoverageBlock> { - self.dominators.as_ref().unwrap() + pub fn rank_partial_cmp( + &self, + a: BasicCoverageBlock, + b: BasicCoverageBlock, + ) -> Option<Ordering> { + self.dominators.as_ref().unwrap().rank_partial_cmp(a, b) } } @@ -650,26 +655,6 @@ pub(super) fn find_loop_backedges( let mut backedges = IndexVec::from_elem_n(Vec::<BasicCoverageBlock>::new(), num_bcbs); // Identify loops by their backedges. - // - // The computational complexity is bounded by: n(s) x d where `n` is the number of - // `BasicCoverageBlock` nodes (the simplified/reduced representation of the CFG derived from the - // MIR); `s` is the average number of successors per node (which is most likely less than 2, and - // independent of the size of the function, so it can be treated as a constant); - // and `d` is the average number of dominators per node. - // - // The average number of dominators depends on the size and complexity of the function, and - // nodes near the start of the function's control flow graph typically have less dominators - // than nodes near the end of the CFG. Without doing a detailed mathematical analysis, I - // think the resulting complexity has the characteristics of O(n log n). - // - // The overall complexity appears to be comparable to many other MIR transform algorithms, and I - // don't expect that this function is creating a performance hot spot, but if this becomes an - // issue, there may be ways to optimize the `dominates` algorithm (as indicated by an - // existing `FIXME` comment in that code), or possibly ways to optimize it's usage here, perhaps - // by keeping track of results for visited `BasicCoverageBlock`s if they can be used to short - // circuit downstream `dominates` checks. - // - // For now, that kind of optimization seems unnecessarily complicated. for (bcb, _) in basic_coverage_blocks.iter_enumerated() { for &successor in &basic_coverage_blocks.successors[bcb] { if basic_coverage_blocks.dominates(successor, bcb) { diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 287ae217087..d27200419e2 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,4 +1,3 @@ -use super::debug::term_type; use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB}; use itertools::Itertools; @@ -40,7 +39,7 @@ impl CoverageStatement { "{}: @{}.{}: {:?}", source_range_no_file(tcx, span), bb.index(), - term_type(&term.kind), + term.kind.name(), term.kind ) } @@ -345,7 +344,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { // before the dominated equal spans). When later comparing two spans in // order, the first will either dominate the second, or they will have no // dominator relationship. - self.basic_coverage_blocks.dominators().rank_partial_cmp(a.bcb, b.bcb) + self.basic_coverage_blocks.rank_partial_cmp(a.bcb, b.bcb) } } else { // Sort hi() in reverse order so shorter spans are attempted after longer spans. diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 83a335197b3..90b58933df7 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -25,7 +25,6 @@ //! to: `rustc_span::create_default_session_globals_then(|| { test_here(); })`. use super::counters; -use super::debug; use super::graph; use super::spans; @@ -188,12 +187,12 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String { | TerminatorKind::Goto { target } | TerminatorKind::InlineAsm { destination: Some(target), .. } | TerminatorKind::Yield { resume: target, .. } => { - format!("{}{:?}:{} -> {:?}", sp, bb, debug::term_type(kind), target) + format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), target) } TerminatorKind::SwitchInt { targets, .. } => { - format!("{}{:?}:{} -> {:?}", sp, bb, debug::term_type(kind), targets) + format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets) } - _ => format!("{}{:?}:{}", sp, bb, debug::term_type(kind)), + _ => format!("{}{:?}:{}", sp, bb, kind.name()), } }) .collect::<Vec<_>>() @@ -215,7 +214,7 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { " {:?} [label=\"{:?}: {}\"];\n{}", bb, bb, - debug::term_type(&data.terminator().kind), + data.terminator().kind.name(), mir_body .basic_blocks .successors(bb) @@ -244,7 +243,7 @@ fn print_coverage_graphviz( " {:?} [label=\"{:?}: {}\"];\n{}", bcb, bcb, - debug::term_type(&bcb_data.terminator(mir_body).kind), + bcb_data.terminator(mir_body).kind.name(), basic_coverage_blocks .successors(bcb) .map(|successor| { format!(" {:?} -> {:?};", bcb, successor) }) diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs index 1b3ac78fbc6..bf5722b3d00 100644 --- a/compiler/rustc_mir_transform/src/ctfe_limit.rs +++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs @@ -47,7 +47,7 @@ fn has_back_edge( return false; } // Check if any of the dominators of the node are also the node's successor. - doms.dominators(node).any(|dom| node_data.terminator().successors().any(|succ| succ == dom)) + node_data.terminator().successors().any(|succ| doms.dominates(succ, node)) } fn insert_counter(basic_block_data: &mut BasicBlockData<'_>) { diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs index 594cbd8977e..746e3d9652d 100644 --- a/compiler/rustc_mir_transform/src/dump_mir.rs +++ b/compiler/rustc_mir_transform/src/dump_mir.rs @@ -12,7 +12,7 @@ use rustc_session::config::OutputType; pub struct Marker(pub &'static str); impl<'tcx> MirPass<'tcx> for Marker { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.0 } diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 98e7a519c20..fda0e1023f7 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -14,7 +14,7 @@ use rustc_mir_dataflow::un_derefer::UnDerefer; use rustc_mir_dataflow::MoveDataParamEnv; use rustc_mir_dataflow::{on_all_children_bits, on_all_drop_children_bits}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; -use rustc_span::{DesugaringKind, Span}; +use rustc_span::Span; use rustc_target::abi::{FieldIdx, VariantIdx}; use std::fmt; @@ -401,7 +401,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let terminator = data.terminator(); match terminator.kind { - TerminatorKind::Drop { mut place, target, unwind } => { + TerminatorKind::Drop { mut place, target, unwind, replace } => { if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) { place = new_place; } @@ -434,10 +434,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { ) } LookupResult::Parent(..) => { - if !matches!( - terminator.source_info.span.desugaring_kind(), - Some(DesugaringKind::Replace), - ) { + if !replace { self.tcx.sess.delay_span_bug( terminator.source_info.span, format!("drop of untracked value {:?}", bb), diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 602e40d5131..22f71bb0851 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -163,7 +163,14 @@ impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> { self, diag: &'b mut DiagnosticBuilder<'a, ()>, ) -> &'b mut DiagnosticBuilder<'a, ()> { - diag.span_label(self.span(), format!("{:?}", self.panic())); + let span = self.span(); + let assert_kind = self.panic(); + let message = assert_kind.diagnostic_message(); + assert_kind.add_args(&mut |name, value| { + diag.set_arg(name, value); + }); + diag.span_label(span, message); + diag } @@ -191,7 +198,7 @@ impl<P> AssertLint<P> { AssertLint::ArithmeticOverflow(sp, _) | AssertLint::UnconditionalPanic(sp, _) => *sp, } } - pub fn panic(&self) -> &AssertKind<P> { + pub fn panic(self) -> AssertKind<P> { match self { AssertLint::ArithmeticOverflow(_, p) | AssertLint::UnconditionalPanic(_, p) => p, } diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 5989dbebf2d..b1c9c4acc40 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -83,7 +83,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { // If the inner type matches the type bound by `Pointer` if inner_ty == bound_ty { // Do a substitution using the parameters from the callsite - let subst_ty = EarlyBinder(inner_ty).subst(self.tcx, substs_ref); + let subst_ty = EarlyBinder::bind(inner_ty).subst(self.tcx, substs_ref); if let Some((fn_id, fn_substs)) = FunctionItemRefChecker::is_fn_ref(subst_ty) { diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 98f5020e355..fe3f8ed047a 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -1046,7 +1046,10 @@ fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { for (block, block_data) in body.basic_blocks.iter_enumerated() { let (target, unwind, source_info) = match block_data.terminator() { - Terminator { source_info, kind: TerminatorKind::Drop { place, target, unwind } } => { + Terminator { + source_info, + kind: TerminatorKind::Drop { place, target, unwind, replace: _ }, + } => { if let Some(local) = place.as_local() { if local == SELF_ARG { (target, unwind, source_info) @@ -1305,6 +1308,7 @@ fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock { place: Place::from(SELF_ARG), target: return_block, unwind: UnwindAction::Continue, + replace: false, }; let source_info = SourceInfo::outermost(body.span); diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 12f955d46bd..5487b5987e0 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -10,7 +10,6 @@ use rustc_middle::mir::*; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; -use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span}; use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; use rustc_target::spec::abi::Abi; @@ -146,13 +145,16 @@ impl<'tcx> Inliner<'tcx> { Ok(new_blocks) => { debug!("inlined {}", callsite.callee); self.changed = true; + + self.history.push(callsite.callee.def_id()); + self.process_blocks(caller_body, new_blocks); + self.history.pop(); + inlined_count += 1; if inlined_count == inline_limit { + debug!("inline count reached"); return; } - self.history.push(callsite.callee.def_id()); - self.process_blocks(caller_body, new_blocks); - self.history.pop(); } } } @@ -168,6 +170,18 @@ impl<'tcx> Inliner<'tcx> { ) -> Result<std::ops::Range<BasicBlock>, &'static str> { let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); self.check_codegen_attributes(callsite, callee_attrs)?; + + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; + for arg in args { + if !arg.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.param_env) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("Call has unsized argument"); + } + } + self.check_mir_is_available(caller_body, &callsite.callee)?; let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; self.check_mir_body(callsite, callee_body, callee_attrs)?; @@ -181,7 +195,7 @@ impl<'tcx> Inliner<'tcx> { let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, - ty::EarlyBinder(callee_body.clone()), + ty::EarlyBinder::bind(callee_body.clone()), ) else { return Err("failed to normalize callee body"); }; @@ -189,9 +203,6 @@ impl<'tcx> Inliner<'tcx> { // Check call signature compatibility. // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; let output_type = callee_body.return_ty(); if !util::is_subtype(self.tcx, self.param_env, output_type, destination_ty) { trace!(?output_type, ?destination_ty); @@ -441,16 +452,16 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); - if let TerminatorKind::Drop { ref place, target, unwind } = term.kind { + if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. let ty = callsite .callee - .subst_mir(self.tcx, ty::EarlyBinder(&place.ty(callee_body, tcx).ty)); + .subst_mir(self.tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { - work_list.push(unwind); - } + work_list.push(unwind); + } } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { @@ -542,16 +553,6 @@ impl<'tcx> Inliner<'tcx> { // Copy the arguments if needed. let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, &callee_body); - let mut expn_data = ExpnData::default( - ExpnKind::Inlined, - callsite.source_info.span, - self.tcx.sess.edition(), - None, - None, - ); - expn_data.def_site = callee_body.span; - let expn_data = - self.tcx.with_stable_hashing_context(|hcx| LocalExpnId::fresh(expn_data, hcx)); let mut integrator = Integrator { args: &args, new_locals: Local::new(caller_body.local_decls.len()).., @@ -563,7 +564,6 @@ impl<'tcx> Inliner<'tcx> { cleanup_block: unwind, in_cleanup_block: false, tcx: self.tcx, - expn_data, always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), }; @@ -793,7 +793,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = self .instance - .subst_mir(tcx, ty::EarlyBinder(&place.ty(self.callee_body, tcx).ty)); + .subst_mir(tcx, ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -804,7 +804,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder(&f.literal.ty())); + let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder::bind(&f.literal.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST @@ -947,7 +947,6 @@ struct Integrator<'a, 'tcx> { cleanup_block: UnwindAction, in_cleanup_block: bool, tcx: TyCtxt<'tcx>, - expn_data: LocalExpnId, always_live_locals: BitSet<Local>, } @@ -1033,11 +1032,6 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *scope = self.map_scope(*scope); } - fn visit_span(&mut self, span: &mut Span) { - // Make sure that all spans track the fact that they were inlined. - *span = span.fresh_expansion(self.expn_data); - } - fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { self.in_cleanup_block = data.is_cleanup; self.super_basic_block_data(block, data); diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 1ccf06f6153..8a10445f837 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -47,7 +47,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( let Ok(substs) = caller.try_subst_mir_and_normalize_erasing_regions( tcx, param_env, - ty::EarlyBinder(substs), + ty::EarlyBinder::bind(substs), ) else { trace!(?caller, ?param_env, ?substs, "cannot normalize, skipping"); continue; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 65864dc016f..7d9f6c38e36 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -3,6 +3,7 @@ #![deny(rustc::diagnostic_outside_of_impl)] #![feature(box_patterns)] #![feature(drain_filter)] +#![feature(is_sorted)] #![feature(let_chains)] #![feature(map_try_insert)] #![feature(min_specialization)] @@ -84,6 +85,7 @@ mod match_branches; mod multiple_return_terminators; mod normalize_array_len; mod nrvo; +mod prettify; mod ref_prop; mod remove_noop_landing_pads; mod remove_storage_markers; @@ -557,10 +559,13 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) &multiple_return_terminators::MultipleReturnTerminators, &instsimplify::InstSimplify, - &separate_const_switch::SeparateConstSwitch, &simplify::SimplifyLocals::BeforeConstProp, ©_prop::CopyProp, &ref_prop::ReferencePropagation, + // Perform `SeparateConstSwitch` after SSA-based analyses, as cloning blocks may + // destroy the SSA property. It should still happen before const-propagation, so the + // latter pass will leverage the created opportunities. + &separate_const_switch::SeparateConstSwitch, &const_prop::ConstProp, &dataflow_const_prop::DataflowConstProp, // @@ -581,6 +586,9 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &large_enums::EnumSizeOpt { discrepancy: 128 }, // Some cleanup necessary at least for LLVM and potentially other codegen backends. &add_call_guards::CriticalCallEdges, + // Cleanup for human readability, off by default. + &prettify::ReorderBasicBlocks, + &prettify::ReorderLocals, // Dump the end result for testing and debugging purposes. &dump_mir::Marker("PreCodegen"), ], diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index dae01e41e5f..3a7d58f7125 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -82,30 +82,35 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { drop(args); terminator.kind = TerminatorKind::Goto { target }; } - sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => { - if let Some(target) = *target { - let lhs; - let rhs; - { - let mut args = args.drain(..); - lhs = args.next().unwrap(); - rhs = args.next().unwrap(); - } - let bin_op = match intrinsic_name { - sym::wrapping_add => BinOp::Add, - sym::wrapping_sub => BinOp::Sub, - sym::wrapping_mul => BinOp::Mul, - _ => bug!("unexpected intrinsic"), - }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( - *destination, - Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))), - ))), - }); - terminator.kind = TerminatorKind::Goto { target }; + sym::wrapping_add + | sym::wrapping_sub + | sym::wrapping_mul + | sym::unchecked_div + | sym::unchecked_rem => { + let target = target.unwrap(); + let lhs; + let rhs; + { + let mut args = args.drain(..); + lhs = args.next().unwrap(); + rhs = args.next().unwrap(); } + let bin_op = match intrinsic_name { + sym::wrapping_add => BinOp::Add, + sym::wrapping_sub => BinOp::Sub, + sym::wrapping_mul => BinOp::Mul, + sym::unchecked_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + _ => bug!("unexpected intrinsic"), + }; + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + *destination, + Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))), + ))), + }); + terminator.kind = TerminatorKind::Goto { target }; } sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { if let Some(target) = *target { diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 59942dc76f9..6eb48498274 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -41,7 +41,7 @@ pub struct MatchBranchSimplification; impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 + sess.mir_opt_level() >= 1 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -62,7 +62,12 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { .. } if targets.iter().len() == 1 => { let (value, target) = targets.iter().next().unwrap(); - if target == targets.otherwise() { + // We require that this block and the two possible target blocks all be + // distinct. + if target == targets.otherwise() + || bb_idx == target + || bb_idx == targets.otherwise() + { continue; } (discr, value, target, targets.otherwise()) diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 85b26220b1e..5ce96012b90 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -108,7 +108,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { // If multiple different locals are copied to the return place. We can't pick a // single one to rename. - if copied_to_return_place.map_or(false, |old| old != returned_local) { + if copied_to_return_place.is_some_and(|old| old != returned_local) { return None; } diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index e1b65823a5a..710eed3ed38 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -6,7 +6,7 @@ use crate::{validate, MirPass}; /// Just like `MirPass`, except it cannot mutate `Body`. pub trait MirLint<'tcx> { - fn name(&self) -> &str { + fn name(&self) -> &'static str { let name = std::any::type_name::<Self>(); if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name } } @@ -26,7 +26,7 @@ impl<'tcx, T> MirPass<'tcx> for Lint<T> where T: MirLint<'tcx>, { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.0.name() } @@ -49,7 +49,7 @@ impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T> where T: MirPass<'tcx>, { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.1.name() } @@ -121,7 +121,7 @@ fn run_passes_inner<'tcx>( validate_body(tcx, body, format!("before pass {}", name)); } - pass.run_pass(tcx, body); + tcx.sess.time(name, || pass.run_pass(tcx, body)); if dump_enabled { dump_mir_for_pass(tcx, body, &name, true); diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs new file mode 100644 index 00000000000..6f46974ea00 --- /dev/null +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -0,0 +1,150 @@ +//! These two passes provide no value to the compiler, so are off at every level. +//! +//! However, they can be enabled on the command line +//! (`-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals`) +//! to make the MIR easier to read for humans. + +use crate::MirPass; +use rustc_index::{bit_set::BitSet, IndexSlice, IndexVec}; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_session::Session; + +/// Rearranges the basic blocks into a *reverse post-order*. +/// +/// Thus after this pass, all the successors of a block are later than it in the +/// `IndexVec`, unless that successor is a back-edge (such as from a loop). +pub struct ReorderBasicBlocks; + +impl<'tcx> MirPass<'tcx> for ReorderBasicBlocks { + fn is_enabled(&self, _session: &Session) -> bool { + false + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let rpo: IndexVec<BasicBlock, BasicBlock> = + body.basic_blocks.postorder().iter().copied().rev().collect(); + if rpo.iter().is_sorted() { + return; + } + + let mut updater = BasicBlockUpdater { map: rpo.invert_bijective_mapping(), tcx }; + debug_assert_eq!(updater.map[START_BLOCK], START_BLOCK); + updater.visit_body(body); + + permute(body.basic_blocks.as_mut(), &updater.map); + } +} + +/// Rearranges the locals into *use* order. +/// +/// Thus after this pass, a local with a smaller [`Location`] where it was first +/// assigned or referenced will have a smaller number. +/// +/// (Does not reorder arguments nor the [`RETURN_PLACE`].) +pub struct ReorderLocals; + +impl<'tcx> MirPass<'tcx> for ReorderLocals { + fn is_enabled(&self, _session: &Session) -> bool { + false + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut finder = + LocalFinder { map: IndexVec::new(), seen: BitSet::new_empty(body.local_decls.len()) }; + + // We can't reorder the return place or the arguments + for local in (0..=body.arg_count).map(Local::from_usize) { + finder.track(local); + } + + for (bb, bbd) in body.basic_blocks.iter_enumerated() { + finder.visit_basic_block_data(bb, bbd); + } + + // track everything in case there are some locals that we never saw, + // such as in non-block things like debug info or in non-uses. + for local in body.local_decls.indices() { + finder.track(local); + } + + if finder.map.iter().is_sorted() { + return; + } + + let mut updater = LocalUpdater { map: finder.map.invert_bijective_mapping(), tcx }; + + for local in (0..=body.arg_count).map(Local::from_usize) { + debug_assert_eq!(updater.map[local], local); + } + + updater.visit_body_preserves_cfg(body); + + permute(&mut body.local_decls, &updater.map); + } +} + +fn permute<I: rustc_index::Idx + Ord, T>(data: &mut IndexVec<I, T>, map: &IndexSlice<I, I>) { + // FIXME: It would be nice to have a less-awkward way to apply permutations, + // but I don't know one that exists. `sort_by_cached_key` has logic for it + // internally, but not in a way that we're allowed to use here. + let mut enumerated: Vec<_> = std::mem::take(data).into_iter_enumerated().collect(); + enumerated.sort_by_key(|p| map[p.0]); + *data = enumerated.into_iter().map(|p| p.1).collect(); +} + +struct BasicBlockUpdater<'tcx> { + map: IndexVec<BasicBlock, BasicBlock>, + tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for BasicBlockUpdater<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, _location: Location) { + for succ in terminator.successors_mut() { + *succ = self.map[*succ]; + } + } +} + +struct LocalFinder { + map: IndexVec<Local, Local>, + seen: BitSet<Local>, +} + +impl LocalFinder { + fn track(&mut self, l: Local) { + if self.seen.insert(l) { + self.map.push(l); + } + } +} + +impl<'tcx> Visitor<'tcx> for LocalFinder { + fn visit_local(&mut self, l: Local, context: PlaceContext, _location: Location) { + // Exclude non-uses to keep `StorageLive` from controlling where we put + // a `Local`, since it might not actually be assigned until much later. + if context.is_use() { + self.track(l); + } + } +} + +struct LocalUpdater<'tcx> { + pub map: IndexVec<Local, Local>, + pub tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { + *l = self.map[*l]; + } +} diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 0ea8f2ba93f..243cb463560 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -17,8 +17,8 @@ impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { let literal = constant.literal; match literal { ConstantKind::Ty(c) => match c.kind() { - ConstKind::Param(_) | ConstKind::Error(_) => {} - _ => bug!("only ConstKind::Param should be encountered here, got {:#?}", c), + ConstKind::Param(_) | ConstKind::Error(_) | ConstKind::Value(_) => {} + _ => bug!("only ConstKind::Param/Value should be encountered here, got {:#?}", c), }, ConstantKind::Unevaluated(..) => self.required_consts.push(*constant), ConstantKind::Val(..) => {} diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 2479856b727..f35a5fb4276 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -46,7 +46,7 @@ pub struct SeparateConstSwitch; impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 4 + sess.mir_opt_level() >= 2 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 7c47d8814db..5f12f1937c0 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -69,7 +69,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' // of this function. Is this intentional? if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(Ty::kind) { let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); - let body = EarlyBinder(body.clone()).subst(tcx, substs); + let body = EarlyBinder::bind(body.clone()).subst(tcx, substs); debug!("make_shim({:?}) = {:?}", instance, body); return body; } @@ -544,6 +544,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { place: dest_field, target: unwind, unwind: UnwindAction::Terminate, + replace: false, }, true, ); @@ -643,8 +644,11 @@ fn build_call_shim<'tcx>( let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig)); assert_eq!(sig_substs.is_some(), !instance.has_polymorphic_mir_body()); - let mut sig = - if let Some(sig_substs) = sig_substs { sig.subst(tcx, &sig_substs) } else { sig.0 }; + let mut sig = if let Some(sig_substs) = sig_substs { + sig.subst(tcx, &sig_substs) + } else { + sig.subst_identity() + }; if let CallKind::Indirect(fnty) = call_kind { // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This @@ -800,6 +804,7 @@ fn build_call_shim<'tcx>( place: rcvr_place(), target: BasicBlock::new(2), unwind: UnwindAction::Continue, + replace: false, }, false, ); @@ -815,6 +820,7 @@ fn build_call_shim<'tcx>( place: rcvr_place(), target: BasicBlock::new(4), unwind: UnwindAction::Terminate, + replace: false, }, true, ); diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 1b96df3aed5..e59219321b7 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -74,7 +74,7 @@ pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } impl<'tcx> MirPass<'tcx> for SimplifyCfg { - fn name(&self) -> &str { + fn name(&self) -> &'static str { &self.name() } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 2d77291293d..e4b3b8b9262 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -6,23 +6,29 @@ use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields}; -use rustc_target::abi::FieldIdx; +use rustc_target::abi::{FieldIdx, ReprFlags, FIRST_VARIANT}; pub struct ScalarReplacementOfAggregates; impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 + sess.mir_opt_level() >= 2 } #[instrument(level = "debug", skip(self, tcx, body))] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!(def_id = ?body.source.def_id()); + + // Avoid query cycles (generators require optimized MIR for layout). + if tcx.type_of(body.source.def_id()).subst_identity().is_generator() { + return; + } + let mut excluded = excluded_locals(body); let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); loop { debug!(?excluded); - let escaping = escaping_locals(&excluded, body); + let escaping = escaping_locals(tcx, param_env, &excluded, body); debug!(?escaping); let replacements = compute_flattening(tcx, param_env, body, escaping); debug!(?replacements); @@ -48,11 +54,45 @@ impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates { /// - the locals is a union or an enum; /// - the local's address is taken, and thus the relative addresses of the fields are observable to /// client code. -fn escaping_locals(excluded: &BitSet<Local>, body: &Body<'_>) -> BitSet<Local> { +fn escaping_locals<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + excluded: &BitSet<Local>, + body: &Body<'tcx>, +) -> BitSet<Local> { + let is_excluded_ty = |ty: Ty<'tcx>| { + if ty.is_union() || ty.is_enum() { + return true; + } + if let ty::Adt(def, _substs) = ty.kind() { + if def.repr().flags.contains(ReprFlags::IS_SIMD) { + // Exclude #[repr(simd)] types so that they are not de-optimized into an array + return true; + } + // We already excluded unions and enums, so this ADT must have one variant + let variant = def.variant(FIRST_VARIANT); + if variant.fields.len() > 1 { + // If this has more than one field, it cannot be a wrapper that only provides a + // niche, so we do not want to automatically exclude it. + return false; + } + let Ok(layout) = tcx.layout_of(param_env.and(ty)) else { + // We can't get the layout + return true; + }; + if layout.layout.largest_niche().is_some() { + // This type has a niche + return true; + } + } + // Default for non-ADTs + false + }; + let mut set = BitSet::new_empty(body.local_decls.len()); set.insert_range(RETURN_PLACE..=Local::from_usize(body.arg_count)); for (local, decl) in body.local_decls().iter_enumerated() { - if decl.ty.is_union() || decl.ty.is_enum() || excluded.contains(local) { + if excluded.contains(local) || is_excluded_ty(decl.ty) { set.insert(local); } } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 2b404efccc7..7a0d3a025f3 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -31,11 +31,11 @@ pub struct SsaLocals { /// We often encounter MIR bodies with 1 or 2 basic blocks. In those cases, it's unnecessary to /// actually compute dominators, we can just compare block indices because bb0 is always the first /// block, and in any body all other blocks are always dominated by bb0. -struct SmallDominators { - inner: Option<Dominators<BasicBlock>>, +struct SmallDominators<'a> { + inner: Option<&'a Dominators<BasicBlock>>, } -impl SmallDominators { +impl SmallDominators<'_> { fn dominates(&self, first: Location, second: Location) -> bool { if first.block == second.block { first.statement_index <= second.statement_index @@ -198,14 +198,14 @@ enum LocationExtended { Arg, } -struct SsaVisitor { - dominators: SmallDominators, +struct SsaVisitor<'a> { + dominators: SmallDominators<'a>, assignments: IndexVec<Local, Set1<LocationExtended>>, assignment_order: Vec<Local>, direct_uses: IndexVec<Local, u32>, } -impl<'tcx> Visitor<'tcx> for SsaVisitor { +impl<'tcx> Visitor<'tcx> for SsaVisitor<'_> { fn visit_local(&mut self, local: Local, ctxt: PlaceContext, loc: Location) { match ctxt { PlaceContext::MutatingUse(MutatingUseContext::Projection) @@ -216,7 +216,6 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor { PlaceContext::NonMutatingUse( NonMutatingUseContext::SharedBorrow | NonMutatingUseContext::ShallowBorrow - | NonMutatingUseContext::UniqueBorrow | NonMutatingUseContext::AddressOf, ) | PlaceContext::MutatingUse(_) => { |