diff options
Diffstat (limited to 'compiler')
39 files changed, 692 insertions, 145 deletions
diff --git a/compiler/rustc_codegen_gcc/src/debuginfo.rs b/compiler/rustc_codegen_gcc/src/debuginfo.rs index 4c8585192a1..0f015cc23f5 100644 --- a/compiler/rustc_codegen_gcc/src/debuginfo.rs +++ b/compiler/rustc_codegen_gcc/src/debuginfo.rs @@ -29,13 +29,24 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> { _variable_alloca: Self::Value, _direct_offset: Size, _indirect_offsets: &[Size], - _fragment: Option<Range<Size>>, + _fragment: &Option<Range<Size>>, ) { // FIXME(tempdragon): Not sure if this is correct, probably wrong but still keep it here. #[cfg(feature = "master")] _variable_alloca.set_location(_dbg_loc); } + fn dbg_var_value( + &mut self, + _dbg_var: Self::DIVariable, + _dbg_loc: Self::DILocation, + _value: Self::Value, + _direct_offset: Size, + _indirect_offsets: &[Size], + _fragment: &Option<Range<Size>>, + ) { + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { // TODO(antoyo): insert reference to gdb debug scripts section global. } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs index 40842915222..52d04625749 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs @@ -35,3 +35,6 @@ declare_constant!(DW_OP_plus_uconst: u64); /// Double-checked by a static assertion in `RustWrapper.cpp`. #[allow(non_upper_case_globals)] pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000; +// It describes the actual value of a source variable which might not exist in registers or in memory. +#[allow(non_upper_case_globals)] +pub(crate) const DW_OP_stack_value: u64 = 0x9f; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index af64e4ebed0..c6ad1c2e18e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -156,7 +156,7 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], - fragment: Option<Range<Size>>, + fragment: &Option<Range<Size>>, ) { use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst}; @@ -187,7 +187,6 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) }; unsafe { - // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( di_builder, variable_alloca, @@ -199,6 +198,56 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { }; } + fn dbg_var_value( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + value: Self::Value, + direct_offset: Size, + indirect_offsets: &[Size], + fragment: &Option<Range<Size>>, + ) { + use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst, DW_OP_stack_value}; + + // Convert the direct and indirect offsets and fragment byte range to address ops. + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + + if direct_offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(DW_OP_stack_value); + } + for &offset in indirect_offsets { + addr_ops.push(DW_OP_deref); + if offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(offset.bytes() as u64); + } + } + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(DW_OP_LLVM_fragment); + addr_ops.push(fragment.start.bits() as u64); + addr_ops.push((fragment.end - fragment.start).bits() as u64); + } + + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; + unsafe { + llvm::LLVMDIBuilderInsertDbgValueRecordAtEnd( + di_builder, + value, + dbg_var, + addr_expr, + dbg_loc, + self.llbb(), + ); + } + } + fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { unsafe { llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e9f92267a7d..7fbba029407 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1991,6 +1991,15 @@ unsafe extern "C" { Block: &'ll BasicBlock, ) -> &'ll DbgRecord; + pub(crate) fn LLVMDIBuilderInsertDbgValueRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Val: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( Builder: &DIBuilder<'ll>, Scope: &'ll Metadata, diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 45bc5451946..0a37a904193 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -260,6 +260,10 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer PlaceContext::MutatingUse(MutatingUseContext::Yield) => bug!(), } } + + fn visit_statement_debuginfo(&mut self, _: &mir::StmtDebugInfo<'tcx>, _: Location) { + // Debuginfo does not generate actual code. + } } #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index b2dc4fe32b0..e371f1a7623 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1320,6 +1320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { for statement in &data.statements { self.codegen_statement(bx, statement); } + self.codegen_stmt_debuginfos(bx, &data.after_last_stmt_debuginfos); let merging_succ = self.codegen_terminator(bx, bb, data.terminator()); if let MergingSucc::False = merging_succ { diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index b8f635ab781..0c9acf087f9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -253,6 +253,53 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { spill_slot } + // Indicates that local is set to a new value. The `layout` and `projection` are used to + // calculate the offset. + pub(crate) fn debug_new_val_to_local( + &self, + bx: &mut Bx, + local: mir::Local, + base: PlaceRef<'tcx, Bx::Value>, + projection: &[mir::PlaceElem<'tcx>], + ) { + let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full; + if !full_debug_info { + return; + } + + let vars = match &self.per_local_var_debug_info { + Some(per_local) => &per_local[local], + None => return, + }; + + let DebugInfoOffset { direct_offset, indirect_offsets, result: _ } = + calculate_debuginfo_offset(bx, projection, base.layout); + for var in vars.iter() { + let Some(dbg_var) = var.dbg_var else { + continue; + }; + let Some(dbg_loc) = self.dbg_loc(var.source_info) else { + continue; + }; + bx.dbg_var_value( + dbg_var, + dbg_loc, + base.val.llval, + direct_offset, + &indirect_offsets, + &var.fragment, + ); + } + } + + pub(crate) fn debug_poison_to_local(&self, bx: &mut Bx, local: mir::Local) { + let ty = self.monomorphize(self.mir.local_decls[local].ty); + let layout = bx.cx().layout_of(ty); + let to_backend_ty = bx.cx().immediate_backend_type(layout); + let place_ref = PlaceRef::new_sized(bx.cx().const_poison(to_backend_ty), layout); + self.debug_new_val_to_local(bx, local, place_ref, &[]); + } + /// Apply debuginfo and/or name, after creating the `alloca` for a local, /// or initializing the local with an operand (whichever applies). pub(crate) fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) { @@ -424,7 +471,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { alloca.val.llval, Size::ZERO, &[Size::ZERO], - var.fragment, + &var.fragment, ); } else { bx.dbg_var_addr( @@ -433,7 +480,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { base.val.llval, direct_offset, &indirect_offsets, - var.fragment, + &var.fragment, ); } } @@ -455,7 +502,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let base = FunctionCx::spill_operand_to_stack(operand, Some(name), bx); bx.clear_dbg_loc(); - bx.dbg_var_addr(dbg_var, dbg_loc, base.val.llval, Size::ZERO, &[], fragment); + bx.dbg_var_addr(dbg_var, dbg_loc, base.val.llval, Size::ZERO, &[], &fragment); } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 5f7f87fc692..88a8e2a844c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -71,16 +71,23 @@ pub enum OperandValue<V> { } impl<V: CodegenObject> OperandValue<V> { + /// Return the data pointer and optional metadata as backend values + /// if this value can be treat as a pointer. + pub(crate) fn try_pointer_parts(self) -> Option<(V, Option<V>)> { + match self { + OperandValue::Immediate(llptr) => Some((llptr, None)), + OperandValue::Pair(llptr, llextra) => Some((llptr, Some(llextra))), + OperandValue::Ref(_) | OperandValue::ZeroSized => None, + } + } + /// Treat this value as a pointer and return the data pointer and /// optional metadata as backend values. /// /// If you're making a place, use [`Self::deref`] instead. pub(crate) fn pointer_parts(self) -> (V, Option<V>) { - match self { - OperandValue::Immediate(llptr) => (llptr, None), - OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), - _ => bug!("OperandValue cannot be a pointer: {self:?}"), - } + self.try_pointer_parts() + .unwrap_or_else(|| bug!("OperandValue cannot be a pointer: {self:?}")) } /// Treat this value as a pointer and return the place to which it points. diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 0a50d7f18db..88590b6271b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::{self, NonDivergingIntrinsic}; +use rustc_middle::mir::{self, NonDivergingIntrinsic, StmtDebugInfo}; use rustc_middle::span_bug; use tracing::instrument; @@ -8,6 +8,7 @@ use crate::traits::*; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[instrument(level = "debug", skip(self, bx))] pub(crate) fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) { + self.codegen_stmt_debuginfos(bx, &statement.debuginfos); self.set_debug_loc(bx, statement.source_info); match statement.kind { mir::StatementKind::Assign(box (ref place, ref rvalue)) => { @@ -101,4 +102,49 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | mir::StatementKind::Nop => {} } } + + pub(crate) fn codegen_stmt_debuginfo(&mut self, bx: &mut Bx, debuginfo: &StmtDebugInfo<'tcx>) { + match debuginfo { + StmtDebugInfo::AssignRef(dest, place) => { + let local_ref = match self.locals[place.local] { + // For an rvalue like `&(_1.1)`, when `BackendRepr` is `BackendRepr::Memory`, we allocate a block of memory to this place. + // The place is an indirect pointer, we can refer to it directly. + LocalRef::Place(place_ref) => Some((place_ref, place.projection.as_slice())), + // For an rvalue like `&((*_1).1)`, we are calculating the address of `_1.1`. + // The deref projection is no-op here. + LocalRef::Operand(operand_ref) if place.is_indirect_first_projection() => { + Some((operand_ref.deref(bx.cx()), &place.projection[1..])) + } + // For an rvalue like `&1`, when `BackendRepr` is `BackendRepr::Scalar`, + // we cannot get the address. + // N.B. `non_ssa_locals` returns that this is an SSA local. + LocalRef::Operand(_) => None, + LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => None, + } + .filter(|(_, projection)| { + // Drop unsupported projections. + projection.iter().all(|p| p.can_use_in_debuginfo()) + }); + if let Some((base, projection)) = local_ref { + self.debug_new_val_to_local(bx, *dest, base, projection); + } else { + // If the address cannot be calculated, use poison to indicate that the value has been optimized out. + self.debug_poison_to_local(bx, *dest); + } + } + StmtDebugInfo::InvalidAssign(local) => { + self.debug_poison_to_local(bx, *local); + } + } + } + + pub(crate) fn codegen_stmt_debuginfos( + &mut self, + bx: &mut Bx, + debuginfos: &[StmtDebugInfo<'tcx>], + ) { + for debuginfo in debuginfos { + self.codegen_stmt_debuginfo(bx, debuginfo); + } + } } diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index b9d4950e0ad..a4da6c915de 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -77,7 +77,19 @@ pub trait DebugInfoBuilderMethods: BackendTypes { indirect_offsets: &[Size], // Byte range in the `dbg_var` covered by this fragment, // if this is a fragment of a composite `DIVariable`. - fragment: Option<Range<Size>>, + fragment: &Option<Range<Size>>, + ); + fn dbg_var_value( + &mut self, + dbg_var: Self::DIVariable, + dbg_loc: Self::DILocation, + value: Self::Value, + direct_offset: Size, + // NB: each offset implies a deref (i.e. they're steps in a pointer chain). + indirect_offsets: &[Size], + // Byte range in the `dbg_var` covered by this fragment, + // if this is a fragment of a composite `DIVariable`. + fragment: &Option<Range<Size>>, ); fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation); fn clear_dbg_loc(&mut self); diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 2b83ea24ac6..e38474f09ff 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -58,6 +58,7 @@ using namespace llvm::object; // This opcode is an LLVM detail that could hypothetically change (?), so // verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM. static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000); +static_assert(dwarf::DW_OP_stack_value == 0x9f); // LLVMAtomicOrdering is already an enum - don't create another // one. diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 28142382b13..8eb7aa71fcd 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -1298,6 +1298,10 @@ pub struct BasicBlockData<'tcx> { /// List of statements in this block. pub statements: Vec<Statement<'tcx>>, + /// All debuginfos happen before the statement. + /// Put debuginfos here when the last statement is eliminated. + pub after_last_stmt_debuginfos: StmtDebugInfos<'tcx>, + /// Terminator for this block. /// /// N.B., this should generally ONLY be `None` during construction. @@ -1325,7 +1329,12 @@ impl<'tcx> BasicBlockData<'tcx> { terminator: Option<Terminator<'tcx>>, is_cleanup: bool, ) -> BasicBlockData<'tcx> { - BasicBlockData { statements, terminator, is_cleanup } + BasicBlockData { + statements, + after_last_stmt_debuginfos: StmtDebugInfos::default(), + terminator, + is_cleanup, + } } /// Accessor for terminator. @@ -1360,6 +1369,36 @@ impl<'tcx> BasicBlockData<'tcx> { self.terminator().successors() } } + + pub fn retain_statements<F>(&mut self, mut f: F) + where + F: FnMut(&Statement<'tcx>) -> bool, + { + // Place debuginfos into the next retained statement, + // this `debuginfos` variable is used to cache debuginfos between two retained statements. + let mut debuginfos = StmtDebugInfos::default(); + self.statements.retain_mut(|stmt| { + let retain = f(stmt); + if retain { + stmt.debuginfos.prepend(&mut debuginfos); + } else { + debuginfos.append(&mut stmt.debuginfos); + } + retain + }); + self.after_last_stmt_debuginfos.prepend(&mut debuginfos); + } + + pub fn strip_nops(&mut self) { + self.retain_statements(|stmt| !matches!(stmt.kind, StatementKind::Nop)) + } + + pub fn drop_debuginfo(&mut self) { + self.after_last_stmt_debuginfos.drop_debuginfo(); + for stmt in self.statements.iter_mut() { + stmt.debuginfos.drop_debuginfo(); + } + } } /////////////////////////////////////////////////////////////////////////// @@ -1664,10 +1703,10 @@ mod size_asserts { use super::*; // tidy-alphabetical-start - static_assert_size!(BasicBlockData<'_>, 128); + static_assert_size!(BasicBlockData<'_>, 152); static_assert_size!(LocalDecl<'_>, 40); static_assert_size!(SourceScopeData<'_>, 64); - static_assert_size!(Statement<'_>, 32); + static_assert_size!(Statement<'_>, 56); static_assert_size!(Terminator<'_>, 96); static_assert_size!(VarDebugInfo<'_>, 88); // tidy-alphabetical-end diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 350d75c2ee7..d87e3abe3b2 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -719,6 +719,11 @@ impl<'de, 'tcx> MirWriter<'de, 'tcx> { let mut current_location = Location { block, statement_index: 0 }; for statement in &data.statements { (self.extra_data)(PassWhere::BeforeLocation(current_location), w)?; + + for debuginfo in statement.debuginfos.iter() { + writeln!(w, "{INDENT}{INDENT}// DBG: {debuginfo:?};")?; + } + let indented_body = format!("{INDENT}{INDENT}{statement:?};"); if self.options.include_extra_comments { writeln!( @@ -749,6 +754,10 @@ impl<'de, 'tcx> MirWriter<'de, 'tcx> { current_location.statement_index += 1; } + for debuginfo in data.after_last_stmt_debuginfos.iter() { + writeln!(w, "{INDENT}{INDENT}// DBG: {debuginfo:?};")?; + } + // Terminator at the bottom. (self.extra_data)(PassWhere::BeforeLocation(current_location), w)?; if data.terminator.is_some() { @@ -829,6 +838,19 @@ impl Debug for Statement<'_> { } } +impl Debug for StmtDebugInfo<'_> { + fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result { + match self { + StmtDebugInfo::AssignRef(local, place) => { + write!(fmt, "{local:?} = &{place:?}") + } + StmtDebugInfo::InvalidAssign(local) => { + write!(fmt, "{local:?} = &?") + } + } + } +} + impl Display for NonDivergingIntrinsic<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index e009fe05b53..f310e1e5762 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -1,5 +1,7 @@ //! Functionality for statements, operands, places, and things that appear in them. +use std::ops; + use tracing::{debug, instrument}; use super::interpret::GlobalAlloc; @@ -15,17 +17,28 @@ use crate::ty::CoroutineArgsExt; pub struct Statement<'tcx> { pub source_info: SourceInfo, pub kind: StatementKind<'tcx>, + /// Some debuginfos appearing before the primary statement. + pub debuginfos: StmtDebugInfos<'tcx>, } impl<'tcx> Statement<'tcx> { /// Changes a statement to a nop. This is both faster than deleting instructions and avoids /// invalidating statement indices in `Location`s. - pub fn make_nop(&mut self) { - self.kind = StatementKind::Nop + pub fn make_nop(&mut self, drop_debuginfo: bool) { + if matches!(self.kind, StatementKind::Nop) { + return; + } + let replaced_stmt = std::mem::replace(&mut self.kind, StatementKind::Nop); + if !drop_debuginfo { + let Some(debuginfo) = replaced_stmt.as_debuginfo() else { + bug!("debuginfo is not yet supported.") + }; + self.debuginfos.push(debuginfo); + } } pub fn new(source_info: SourceInfo, kind: StatementKind<'tcx>) -> Self { - Statement { source_info, kind } + Statement { source_info, kind, debuginfos: StmtDebugInfos::default() } } } @@ -63,6 +76,17 @@ impl<'tcx> StatementKind<'tcx> { _ => None, } } + + pub fn as_debuginfo(&self) -> Option<StmtDebugInfo<'tcx>> { + match self { + StatementKind::Assign(box (place, Rvalue::Ref(_, _, ref_place))) + if let Some(local) = place.as_local() => + { + Some(StmtDebugInfo::AssignRef(local, *ref_place)) + } + _ => None, + } + } } /////////////////////////////////////////////////////////////////////////// @@ -503,6 +527,20 @@ impl<'tcx> PlaceRef<'tcx> { }) } + /// Return the place accessed locals that include the base local. + pub fn accessed_locals(self) -> impl Iterator<Item = Local> { + std::iter::once(self.local).chain(self.projection.iter().filter_map(|proj| match proj { + ProjectionElem::Index(local) => Some(*local), + ProjectionElem::Deref + | ProjectionElem::Field(_, _) + | ProjectionElem::ConstantIndex { .. } + | ProjectionElem::Subslice { .. } + | ProjectionElem::Downcast(_, _) + | ProjectionElem::OpaqueCast(_) + | ProjectionElem::UnwrapUnsafeBinder(_) => None, + })) + } + /// Generates a new place by appending `more_projections` to the existing ones /// and interning the result. pub fn project_deeper( @@ -967,3 +1005,71 @@ impl RawPtrKind { } } } + +#[derive(Default, Debug, Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] +pub struct StmtDebugInfos<'tcx>(Vec<StmtDebugInfo<'tcx>>); + +impl<'tcx> StmtDebugInfos<'tcx> { + pub fn push(&mut self, debuginfo: StmtDebugInfo<'tcx>) { + self.0.push(debuginfo); + } + + pub fn drop_debuginfo(&mut self) { + self.0.clear(); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn prepend(&mut self, debuginfos: &mut Self) { + if debuginfos.is_empty() { + return; + }; + debuginfos.0.append(self); + std::mem::swap(debuginfos, self); + } + + pub fn append(&mut self, debuginfos: &mut Self) { + if debuginfos.is_empty() { + return; + }; + self.0.append(debuginfos); + } + + pub fn extend(&mut self, debuginfos: &Self) { + if debuginfos.is_empty() { + return; + }; + self.0.extend_from_slice(debuginfos); + } + + pub fn retain<F>(&mut self, f: F) + where + F: FnMut(&StmtDebugInfo<'tcx>) -> bool, + { + self.0.retain(f); + } +} + +impl<'tcx> ops::Deref for StmtDebugInfos<'tcx> { + type Target = Vec<StmtDebugInfo<'tcx>>; + + #[inline] + fn deref(&self) -> &Vec<StmtDebugInfo<'tcx>> { + &self.0 + } +} + +impl<'tcx> ops::DerefMut for StmtDebugInfos<'tcx> { + #[inline] + fn deref_mut(&mut self) -> &mut Vec<StmtDebugInfo<'tcx>> { + &mut self.0 + } +} + +#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)] +pub enum StmtDebugInfo<'tcx> { + AssignRef(Local, Place<'tcx>), + InvalidAssign(Local), +} diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index 4034a3a06e9..4249914346c 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -444,6 +444,14 @@ impl<'tcx> Terminator<'tcx> { self.kind.successors() } + /// Return `Some` if all successors are identical. + #[inline] + pub fn identical_successor(&self) -> Option<BasicBlock> { + let mut successors = self.successors(); + let first_succ = successors.next()?; + if successors.all(|succ| first_succ == succ) { Some(first_succ) } else { None } + } + #[inline] pub fn successors_mut<'a>(&'a mut self, f: impl FnMut(&'a mut BasicBlock)) { self.kind.successors_mut(f) diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index f3923477800..9654e189f2e 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -95,6 +95,14 @@ macro_rules! make_mir_visitor { self.super_source_scope_data(scope_data); } + fn visit_statement_debuginfo( + &mut self, + stmt_debuginfo: & $($mutability)? StmtDebugInfo<'tcx>, + location: Location + ) { + self.super_statement_debuginfo(stmt_debuginfo, location); + } + fn visit_statement( &mut self, statement: & $($mutability)? Statement<'tcx>, @@ -301,6 +309,7 @@ macro_rules! make_mir_visitor { { let BasicBlockData { statements, + after_last_stmt_debuginfos, terminator, is_cleanup: _ } = data; @@ -312,8 +321,11 @@ macro_rules! make_mir_visitor { index += 1; } + let location = Location { block, statement_index: index }; + for debuginfo in after_last_stmt_debuginfos as & $($mutability)? [_] { + self.visit_statement_debuginfo(debuginfo, location); + } if let Some(terminator) = terminator { - let location = Location { block, statement_index: index }; self.visit_terminator(terminator, location); } } @@ -376,14 +388,45 @@ macro_rules! make_mir_visitor { } } + fn super_statement_debuginfo( + &mut self, + stmt_debuginfo: & $($mutability)? StmtDebugInfo<'tcx>, + location: Location + ) { + match stmt_debuginfo { + StmtDebugInfo::AssignRef(local, place) => { + self.visit_local( + $(& $mutability)? *local, + PlaceContext::NonUse(NonUseContext::VarDebugInfo), + location + ); + self.visit_place( + place, + PlaceContext::NonUse(NonUseContext::VarDebugInfo), + location + ); + }, + StmtDebugInfo::InvalidAssign(local) => { + self.visit_local( + $(& $mutability)? *local, + PlaceContext::NonUse(NonUseContext::VarDebugInfo), + location + ); + } + } + } + fn super_statement( &mut self, statement: & $($mutability)? Statement<'tcx>, location: Location ) { - let Statement { source_info, kind } = statement; + let Statement { source_info, kind, debuginfos } = statement; self.visit_source_info(source_info); + for debuginfo in debuginfos as & $($mutability)? [_] { + self.visit_statement_debuginfo(debuginfo, location); + } match kind { StatementKind::Assign(box (place, rvalue)) => { self.visit_assign(place, rvalue, location); diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index 5eba474a60c..f6aaa65ad9f 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -210,6 +210,7 @@ impl DefUse { /// All of the caveats of `MaybeLiveLocals` apply. pub struct MaybeTransitiveLiveLocals<'a> { always_live: &'a DenseBitSet<Local>, + debuginfo_locals: &'a DenseBitSet<Local>, } impl<'a> MaybeTransitiveLiveLocals<'a> { @@ -217,8 +218,48 @@ impl<'a> MaybeTransitiveLiveLocals<'a> { /// considered live. /// /// This should include at least all locals that are ever borrowed. - pub fn new(always_live: &'a DenseBitSet<Local>) -> Self { - MaybeTransitiveLiveLocals { always_live } + pub fn new( + always_live: &'a DenseBitSet<Local>, + debuginfo_locals: &'a DenseBitSet<Local>, + ) -> Self { + MaybeTransitiveLiveLocals { always_live, debuginfo_locals } + } + + pub fn can_be_removed_if_dead<'tcx>( + stmt_kind: &StatementKind<'tcx>, + always_live: &DenseBitSet<Local>, + debuginfo_locals: &'a DenseBitSet<Local>, + ) -> Option<Place<'tcx>> { + // Compute the place that we are storing to, if any + let destination = match stmt_kind { + StatementKind::Assign(box (place, rvalue)) => (rvalue.is_safe_to_remove() + // FIXME: We are not sure how we should represent this debugging information for some statements, + // keep it for now. + && (!debuginfo_locals.contains(place.local) + || (place.as_local().is_some() && stmt_kind.as_debuginfo().is_some()))) + .then_some(*place), + StatementKind::SetDiscriminant { place, .. } | StatementKind::Deinit(place) => { + (!debuginfo_locals.contains(place.local)).then_some(**place) + } + StatementKind::FakeRead(_) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Retag(..) + | StatementKind::AscribeUserType(..) + | StatementKind::PlaceMention(..) + | StatementKind::Coverage(..) + | StatementKind::Intrinsic(..) + | StatementKind::ConstEvalCounter + | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::Nop => None, + }; + if let Some(destination) = destination + && !destination.is_indirect() + && !always_live.contains(destination.local) + { + return Some(destination); + } + None } } @@ -243,32 +284,12 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> { statement: &mir::Statement<'tcx>, location: Location, ) { - // Compute the place that we are storing to, if any - let destination = match &statement.kind { - StatementKind::Assign(assign) => assign.1.is_safe_to_remove().then_some(assign.0), - StatementKind::SetDiscriminant { place, .. } | StatementKind::Deinit(place) => { - Some(**place) - } - StatementKind::FakeRead(_) - | StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Retag(..) - | StatementKind::AscribeUserType(..) - | StatementKind::PlaceMention(..) - | StatementKind::Coverage(..) - | StatementKind::Intrinsic(..) - | StatementKind::ConstEvalCounter - | StatementKind::BackwardIncompatibleDropHint { .. } - | StatementKind::Nop => None, - }; - if let Some(destination) = destination { - if !destination.is_indirect() - && !state.contains(destination.local) - && !self.always_live.contains(destination.local) - { - // This store is dead - return; - } + if let Some(destination) = + Self::can_be_removed_if_dead(&statement.kind, &self.always_live, &self.debuginfo_locals) + && !state.contains(destination.local) + { + // This store is dead + return; } TransferFunction(state).visit_statement(statement, location); } diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs index 4be67b873f7..b0bf7f484be 100644 --- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs +++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs @@ -36,7 +36,9 @@ impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck { CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. }, ) | StatementKind::FakeRead(..) - | StatementKind::BackwardIncompatibleDropHint { .. } => statement.make_nop(), + | StatementKind::BackwardIncompatibleDropHint { .. } => { + statement.make_nop(true) + } StatementKind::Assign(box ( _, Rvalue::Cast( diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index cddeefca681..f0bc286a940 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -138,7 +138,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind && self.storage_to_remove.contains(l) { - stmt.make_nop(); + stmt.make_nop(true); return; } @@ -150,7 +150,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { *rhs && lhs == rhs { - stmt.make_nop(); + stmt.make_nop(true); } } } diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index c5cd06f170c..814eded910d 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -411,7 +411,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = s.kind && self.remap.contains(l) { - s.make_nop(); + s.make_nop(true); } } diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index eea2b0990d7..732c3dcd44a 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -22,21 +22,22 @@ use rustc_mir_dataflow::impls::{ LivenessTransferFunction, MaybeTransitiveLiveLocals, borrowed_locals, }; +use crate::simplify::UsedInStmtLocals; use crate::util::is_within_packed; /// Performs the optimization on the body /// /// The `borrowed` set must be a `DenseBitSet` of all the locals that are ever borrowed in this /// body. It can be generated via the [`borrowed_locals`] function. -fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +/// Returns true if any instruction is eliminated. +fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { let borrowed_locals = borrowed_locals(body); // If the user requests complete debuginfo, mark the locals that appear in it as live, so // we don't remove assignments to them. - let mut always_live = debuginfo_locals(body); - always_live.union(&borrowed_locals); + let debuginfo_locals = debuginfo_locals(body); - let mut live = MaybeTransitiveLiveLocals::new(&always_live) + let mut live = MaybeTransitiveLiveLocals::new(&borrowed_locals, &debuginfo_locals) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -75,47 +76,36 @@ fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } for (statement_index, statement) in bb_data.statements.iter().enumerate().rev() { - let loc = Location { block: bb, statement_index }; - if let StatementKind::Assign(assign) = &statement.kind { - if !assign.1.is_safe_to_remove() { - continue; - } - } - match &statement.kind { - StatementKind::Assign(box (place, _)) - | StatementKind::SetDiscriminant { place: box place, .. } - | StatementKind::Deinit(box place) => { - if !place.is_indirect() && !always_live.contains(place.local) { - live.seek_before_primary_effect(loc); - if !live.get().contains(place.local) { - patch.push(loc); - } - } - } - StatementKind::Retag(_, _) - | StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Coverage(_) - | StatementKind::Intrinsic(_) - | StatementKind::ConstEvalCounter - | StatementKind::PlaceMention(_) - | StatementKind::BackwardIncompatibleDropHint { .. } - | StatementKind::Nop => {} - - StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { - bug!("{:?} not found in this MIR phase!", statement.kind) + if let Some(destination) = MaybeTransitiveLiveLocals::can_be_removed_if_dead( + &statement.kind, + &borrowed_locals, + &debuginfo_locals, + ) { + let loc = Location { block: bb, statement_index }; + live.seek_before_primary_effect(loc); + if !live.get().contains(destination.local) { + let drop_debuginfo = !debuginfo_locals.contains(destination.local); + // When eliminating a dead statement, we need to address + // the debug information for that statement. + assert!( + drop_debuginfo || statement.kind.as_debuginfo().is_some(), + "don't know how to retain the debug information for {:?}", + statement.kind + ); + patch.push((loc, drop_debuginfo)); } } } } if patch.is_empty() && call_operands_to_move.is_empty() { - return; + return false; } + let eliminated = !patch.is_empty(); let bbs = body.basic_blocks.as_mut_preserves_cfg(); - for Location { block, statement_index } in patch { - bbs[block].statements[statement_index].make_nop(); + for (Location { block, statement_index }, drop_debuginfo) in patch { + bbs[block].statements[statement_index].make_nop(drop_debuginfo); } for (block, argument_index) in call_operands_to_move { let TerminatorKind::Call { ref mut args, .. } = bbs[block].terminator_mut().kind else { @@ -125,6 +115,8 @@ fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let Operand::Copy(place) = *arg else { bug!() }; *arg = Operand::Move(place); } + + eliminated } pub(super) enum DeadStoreElimination { @@ -145,7 +137,12 @@ impl<'tcx> crate::MirPass<'tcx> for DeadStoreElimination { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - eliminate(tcx, body); + if eliminate(tcx, body) { + UsedInStmtLocals::new(body).remove_unused_storage_annotations(body); + for data in body.basic_blocks.as_mut_preserves_cfg() { + data.strip_nops(); + } + } } fn is_required(&self) -> bool { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 74c22ff10c1..1f38433fa5a 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -276,7 +276,7 @@ impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> { StatementKind::StorageDead(local) | StatementKind::StorageLive(local) if self.merged_locals.contains(*local) => { - statement.make_nop(); + statement.make_nop(true); return; } _ => (), @@ -291,7 +291,7 @@ impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> { // (this includes the original statement we wanted to eliminate). if dest == place { debug!("{:?} turned into self-assignment, deleting", location); - statement.make_nop(); + statement.make_nop(true); } } _ => {} diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 99691d9e045..3ff8dc6dbb3 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -1877,7 +1877,7 @@ impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> { StatementKind::StorageLive(l) | StatementKind::StorageDead(l) if self.reused_locals.contains(l) => { - stmt.make_nop() + stmt.make_nop(true) } _ => self.super_statement(stmt, loc), } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 3d49eb4e8ef..8593e25d6aa 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -21,7 +21,7 @@ use tracing::{debug, instrument, trace, trace_span}; use crate::cost_checker::{CostChecker, is_call_like}; use crate::deref_separator::deref_finder; -use crate::simplify::simplify_cfg; +use crate::simplify::{UsedInStmtLocals, simplify_cfg}; use crate::validate::validate_types; use crate::{check_inline, util}; @@ -935,7 +935,7 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( in_cleanup_block: false, return_block, tcx, - always_live_locals: DenseBitSet::new_filled(callee_body.local_decls.len()), + always_live_locals: UsedInStmtLocals::new(&callee_body).locals, }; // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones @@ -995,6 +995,10 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( // people working on rust can build with or without debuginfo while // still getting consistent results from the mir-opt tests. caller_body.var_debug_info.append(&mut callee_body.var_debug_info); + } else { + for bb in callee_body.basic_blocks_mut() { + bb.drop_debuginfo(); + } } caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut()); diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 1a91d6bd7da..1b90e9158f6 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -156,7 +156,7 @@ impl<'tcx> crate::MirPass<'tcx> for EnumSizeOpt { patch.add_statement(location, stmt); } - st.make_nop(); + st.make_nop(true); } } diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index cc8ea76011b..2c535d011a0 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -270,7 +270,7 @@ impl<'tcx> MirPatch<'tcx> { body.local_decls.extend(self.new_locals); for loc in self.nop_statements { - bbs[loc.block].statements[loc.statement_index].make_nop(); + bbs[loc.block].statements[loc.statement_index].make_nop(true); } let mut new_statements = self.new_statements; diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 48ddf5a1bca..c7dc18a4a13 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -1049,7 +1049,7 @@ fn promote_candidates<'tcx>( // Eliminate assignments to, and drops of promoted temps. let promoted = |index: Local| temps[index] == TempState::PromotedOut; for block in body.basic_blocks_mut() { - block.statements.retain(|statement| match &statement.kind { + block.retain_statements(|statement| match &statement.kind { StatementKind::Assign(box (place, _)) => { if let Some(index) = place.as_local() { !promoted(index) diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index b9d6e74ecae..deb0a146476 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -435,7 +435,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { StatementKind::StorageLive(l) | StatementKind::StorageDead(l) if self.storage_to_remove.contains(l) => { - stmt.make_nop(); + stmt.make_nop(true); } // Do not remove assignments as they may still be useful for debuginfo. _ => self.super_statement(stmt, loc), diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs index cb598ceb4df..d56b51bb496 100644 --- a/compiler/rustc_mir_transform/src/remove_place_mention.rs +++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs @@ -14,7 +14,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemovePlaceMention { fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running RemovePlaceMention on {:?}", body.source); for data in body.basic_blocks.as_mut_preserves_cfg() { - data.statements.retain(|statement| match statement.kind { + data.retain_statements(|statement| match statement.kind { StatementKind::PlaceMention(..) | StatementKind::Nop => false, _ => true, }) diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs index 1ae33c00968..cb97d2c865a 100644 --- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs +++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs @@ -14,7 +14,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveStorageMarkers { fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running RemoveStorageMarkers on {:?}", body.source); for data in body.basic_blocks.as_mut_preserves_cfg() { - data.statements.retain(|statement| match statement.kind { + data.retain_statements(|statement| match statement.kind { StatementKind::StorageLive(..) | StatementKind::StorageDead(..) | StatementKind::Nop => false, diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index c4dc8638b26..90c1b3520b9 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -141,7 +141,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { && let ty = place_for_ty.ty(self.local_decls, self.tcx).ty && self.known_to_be_zst(ty) { - statement.make_nop(); + statement.make_nop(true); } else { self.super_statement(statement, loc); } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 75917d23883..8b5efb74205 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -35,10 +35,12 @@ //! pre-"runtime" MIR! use itertools::Itertools as _; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::debuginfo::debuginfo_locals; use rustc_span::DUMMY_SP; use smallvec::SmallVec; use tracing::{debug, trace}; @@ -142,7 +144,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { // statements itself to avoid moving the (relatively) large statements twice. // We do not push the statements directly into the target block (`bb`) as that is slower // due to additional reallocations - let mut merged_blocks = Vec::new(); + let mut merged_blocks: Vec<BasicBlock> = Vec::new(); let mut outer_changed = false; loop { let mut changed = false; @@ -157,8 +159,9 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { let mut terminator = self.basic_blocks[bb].terminator.take().expect("invalid terminator state"); - terminator - .successors_mut(|successor| self.collapse_goto_chain(successor, &mut changed)); + terminator.successors_mut(|successor| { + self.collapse_goto_chain(successor, &mut changed); + }); let mut inner_changed = true; merged_blocks.clear(); @@ -175,10 +178,18 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { if statements_to_merge > 0 { let mut statements = std::mem::take(&mut self.basic_blocks[bb].statements); statements.reserve(statements_to_merge); + let mut parent_bb_last_debuginfos = + std::mem::take(&mut self.basic_blocks[bb].after_last_stmt_debuginfos); for &from in &merged_blocks { + if let Some(stmt) = self.basic_blocks[from].statements.first_mut() { + stmt.debuginfos.prepend(&mut parent_bb_last_debuginfos); + } statements.append(&mut self.basic_blocks[from].statements); + parent_bb_last_debuginfos = + std::mem::take(&mut self.basic_blocks[from].after_last_stmt_debuginfos); } self.basic_blocks[bb].statements = statements; + self.basic_blocks[bb].after_last_stmt_debuginfos = parent_bb_last_debuginfos; } self.basic_blocks[bb].terminator = Some(terminator); @@ -218,10 +229,14 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { // goto chains. We should probably benchmark different sizes. let mut terminators: SmallVec<[_; 1]> = Default::default(); let mut current = *start; + // If each successor has only one predecessor, it's a trivial goto chain. + // We can move all debuginfos to the last basic block. + let mut trivial_goto_chain = true; while let Some(terminator) = self.take_terminator_if_simple_goto(current) { let Terminator { kind: TerminatorKind::Goto { target }, .. } = terminator else { unreachable!(); }; + trivial_goto_chain &= self.pred_count[target] == 1; terminators.push((current, terminator)); current = target; } @@ -233,6 +248,17 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { else { unreachable!(); }; + if trivial_goto_chain { + let mut pred_debuginfos = + std::mem::take(&mut self.basic_blocks[current].after_last_stmt_debuginfos); + let debuginfos = if let Some(stmt) = self.basic_blocks[last].statements.first_mut() + { + &mut stmt.debuginfos + } else { + &mut self.basic_blocks[last].after_last_stmt_debuginfos + }; + debuginfos.prepend(&mut pred_debuginfos); + } *changed |= *target != last; *target = last; debug!("collapsing goto chain from {:?} to {:?}", current, target); @@ -303,7 +329,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { fn strip_nops(&mut self) { for blk in self.basic_blocks.iter_mut() { - blk.statements.retain(|stmt| !matches!(stmt.kind, StatementKind::Nop)) + blk.strip_nops(); } } } @@ -476,17 +502,22 @@ fn make_local_map<V>( /// Keeps track of used & unused locals. struct UsedLocals { increment: bool, - arg_count: u32, use_count: IndexVec<Local, u32>, + always_used: DenseBitSet<Local>, } impl UsedLocals { /// Determines which locals are used & unused in the given body. fn new(body: &Body<'_>) -> Self { + let mut always_used = debuginfo_locals(body); + always_used.insert(RETURN_PLACE); + for arg in body.args_iter() { + always_used.insert(arg); + } let mut this = Self { increment: true, - arg_count: body.arg_count.try_into().unwrap(), use_count: IndexVec::from_elem(0, &body.local_decls), + always_used, }; this.visit_body(body); this @@ -494,10 +525,16 @@ impl UsedLocals { /// Checks if local is used. /// - /// Return place and arguments are always considered used. + /// Return place, arguments, var debuginfo are always considered used. fn is_used(&self, local: Local) -> bool { - trace!("is_used({:?}): use_count: {:?}", local, self.use_count[local]); - local.as_u32() <= self.arg_count || self.use_count[local] != 0 + trace!( + "is_used({:?}): use_count: {:?}, always_used: {}", + local, + self.use_count[local], + self.always_used.contains(local) + ); + // To keep things simple, we don't handle debugging information here, these are in DSE. + self.always_used.contains(local) || self.use_count[local] != 0 } /// Updates the use counts to reflect the removal of given statement. @@ -539,10 +576,10 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { self.super_statement(statement, location); } - StatementKind::ConstEvalCounter | StatementKind::Nop => {} - - StatementKind::StorageLive(_local) | StatementKind::StorageDead(_local) => {} - + StatementKind::ConstEvalCounter + | StatementKind::Nop + | StatementKind::StorageLive(..) + | StatementKind::StorageDead(..) => {} StatementKind::Assign(box (ref place, ref rvalue)) => { if rvalue.is_safe_to_remove() { self.visit_lhs(place, location); @@ -560,7 +597,10 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { } } - fn visit_local(&mut self, local: Local, _ctx: PlaceContext, _location: Location) { + fn visit_local(&mut self, local: Local, ctx: PlaceContext, _location: Location) { + if matches!(ctx, PlaceContext::NonUse(_)) { + return; + } if self.increment { self.use_count[local] += 1; } else { @@ -583,28 +623,26 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod for data in body.basic_blocks.as_mut_preserves_cfg() { // Remove unnecessary StorageLive and StorageDead annotations. - data.statements.retain(|statement| { - let keep = match &statement.kind { + for statement in data.statements.iter_mut() { + let keep_statement = match &statement.kind { StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { used_locals.is_used(*local) } - StatementKind::Assign(box (place, _)) => used_locals.is_used(place.local), - - StatementKind::SetDiscriminant { place, .. } - | StatementKind::BackwardIncompatibleDropHint { place, reason: _ } - | StatementKind::Deinit(place) => used_locals.is_used(place.local), - StatementKind::Nop => false, - _ => true, + StatementKind::Assign(box (place, _)) + | StatementKind::SetDiscriminant { box place, .. } + | StatementKind::BackwardIncompatibleDropHint { box place, .. } + | StatementKind::Deinit(box place) => used_locals.is_used(place.local), + _ => continue, }; - - if !keep { - trace!("removing statement {:?}", statement); - modified = true; - used_locals.statement_removed(statement); + if keep_statement { + continue; } - - keep - }); + trace!("removing statement {:?}", statement); + modified = true; + used_locals.statement_removed(statement); + statement.make_nop(true); + } + data.strip_nops(); } } } @@ -619,7 +657,62 @@ impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { self.tcx } + fn visit_statement_debuginfo( + &mut self, + stmt_debuginfo: &mut StmtDebugInfo<'tcx>, + location: Location, + ) { + match stmt_debuginfo { + StmtDebugInfo::AssignRef(local, place) => { + if place.as_ref().accessed_locals().any(|local| self.map[local].is_none()) { + *stmt_debuginfo = StmtDebugInfo::InvalidAssign(*local); + } + } + StmtDebugInfo::InvalidAssign(_) => {} + } + self.super_statement_debuginfo(stmt_debuginfo, location); + } + fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { *l = self.map[*l].unwrap(); } } + +pub(crate) struct UsedInStmtLocals { + pub(crate) locals: DenseBitSet<Local>, +} + +impl UsedInStmtLocals { + pub(crate) fn new(body: &Body<'_>) -> Self { + let mut this = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + this.visit_body(body); + this + } + + pub(crate) fn remove_unused_storage_annotations<'tcx>(&self, body: &mut Body<'tcx>) { + for data in body.basic_blocks.as_mut_preserves_cfg() { + // Remove unnecessary StorageLive and StorageDead annotations. + for statement in data.statements.iter_mut() { + let keep_statement = match &statement.kind { + StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + self.locals.contains(*local) + } + _ => continue, + }; + if keep_statement { + continue; + } + statement.make_nop(true); + } + } + } +} + +impl<'tcx> Visitor<'tcx> for UsedInStmtLocals { + fn visit_local(&mut self, local: Local, context: PlaceContext, _: Location) { + if matches!(context, PlaceContext::NonUse(_)) { + return; + } + self.locals.insert(local); + } +} diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index c60eb566521..4597439e269 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -76,7 +76,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { // delete comparison statement if it the value being switched on was moved, which means // it can not be user later on if opt.can_remove_bin_op_stmt { - bb.statements[opt.bin_op_stmt_idx].make_nop(); + bb.statements[opt.bin_op_stmt_idx].make_nop(true); } else { // if the integer being compared to a const integral is being moved into the // comparison, e.g `_2 = Eq(move _3, const 'x');` @@ -136,7 +136,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { } for (idx, bb_idx) in storage_deads_to_remove { - body.basic_blocks_mut()[bb_idx].statements[idx].make_nop(); + body.basic_blocks_mut()[bb_idx].statements[idx].make_nop(true); } for (idx, stmt) in storage_deads_to_insert { diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 38769885f36..99f10b8d91d 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -318,7 +318,7 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { for (_, _, fl) in final_locals { self.patch.add_statement(location, StatementKind::StorageLive(fl)); } - statement.make_nop(); + statement.make_nop(true); } return; } @@ -327,7 +327,7 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { for (_, _, fl) in final_locals { self.patch.add_statement(location, StatementKind::StorageDead(fl)); } - statement.make_nop(); + statement.make_nop(true); } return; } @@ -337,7 +337,7 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { self.patch .add_statement(location, StatementKind::Deinit(Box::new(fl.into()))); } - statement.make_nop(); + statement.make_nop(true); return; } } @@ -367,7 +367,7 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { ); } } - statement.make_nop(); + statement.make_nop(true); return; } } @@ -429,7 +429,7 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { StatementKind::Assign(Box::new((new_local.into(), rvalue))), ); } - statement.make_nop(); + statement.make_nop(true); return; } } diff --git a/compiler/rustc_mir_transform/src/strip_debuginfo.rs b/compiler/rustc_mir_transform/src/strip_debuginfo.rs index 9ede8aa79c4..7fec25ccb52 100644 --- a/compiler/rustc_mir_transform/src/strip_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/strip_debuginfo.rs @@ -1,5 +1,6 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::debuginfo::debuginfo_locals; use rustc_session::config::MirStripDebugInfo; /// Conditionally remove some of the VarDebugInfo in MIR. @@ -30,6 +31,22 @@ impl<'tcx> crate::MirPass<'tcx> for StripDebugInfo { if place.local.as_usize() <= body.arg_count && place.local != RETURN_PLACE, ) }); + + let debuginfo_locals = debuginfo_locals(body); + for data in body.basic_blocks.as_mut_preserves_cfg() { + for stmt in data.statements.iter_mut() { + stmt.debuginfos.retain(|debuginfo| match debuginfo { + StmtDebugInfo::AssignRef(local, _) | StmtDebugInfo::InvalidAssign(local) => { + debuginfo_locals.contains(*local) + } + }); + } + data.after_last_stmt_debuginfos.retain(|debuginfo| match debuginfo { + StmtDebugInfo::AssignRef(local, _) | StmtDebugInfo::InvalidAssign(local) => { + debuginfo_locals.contains(*local) + } + }); + } } fn is_required(&self) -> bool { diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 25b46241c52..172672a80fb 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -758,7 +758,8 @@ impl Session { // ELF x86-64 abi, but it can be disabled for some compilation units. // // Typically when we're compiling with `-C panic=abort` we don't need - // `uwtable` because we can't generate any exceptions! + // `uwtable` because we can't generate any exceptions! But note that + // some targets require unwind tables to generate backtraces. // Unwind tables are needed when compiling with `-C panic=unwind`, but // LLVM won't omit unwind tables unless the function is also marked as // `nounwind`, so users are allowed to disable `uwtable` emission. diff --git a/compiler/rustc_target/src/spec/base/android.rs b/compiler/rustc_target/src/spec/base/android.rs index 0426ea44c6a..df2757aaabf 100644 --- a/compiler/rustc_target/src/spec/base/android.rs +++ b/compiler/rustc_target/src/spec/base/android.rs @@ -8,10 +8,6 @@ pub(crate) fn opts() -> TargetOptions { base.tls_model = TlsModel::Emulated; base.has_thread_local = false; base.supported_sanitizers = SanitizerSet::ADDRESS; - // This is for backward compatibility, see https://github.com/rust-lang/rust/issues/49867 - // for context. (At that time, there was no `-C force-unwind-tables`, so the only solution - // was to always emit `uwtable`). - base.default_uwtable = true; base.crt_static_respected = true; base } diff --git a/compiler/rustc_target/src/spec/base/linux.rs b/compiler/rustc_target/src/spec/base/linux.rs index 9982c254eca..26e4590cf5e 100644 --- a/compiler/rustc_target/src/spec/base/linux.rs +++ b/compiler/rustc_target/src/spec/base/linux.rs @@ -12,6 +12,9 @@ pub(crate) fn opts() -> TargetOptions { relro_level: RelroLevel::Full, has_thread_local: true, crt_static_respected: true, + // We want backtraces to work by default and they rely on unwind tables + // (regardless of `-C panic` strategy). + default_uwtable: true, supported_split_debuginfo: Cow::Borrowed(&[ SplitDebuginfo::Packed, SplitDebuginfo::Unpacked, diff --git a/compiler/rustc_target/src/spec/targets/arm_unknown_linux_gnueabihf.rs b/compiler/rustc_target/src/spec/targets/arm_unknown_linux_gnueabihf.rs index a3f5389f0aa..0c711d5e71a 100644 --- a/compiler/rustc_target/src/spec/targets/arm_unknown_linux_gnueabihf.rs +++ b/compiler/rustc_target/src/spec/targets/arm_unknown_linux_gnueabihf.rs @@ -19,6 +19,11 @@ pub(crate) fn target() -> Target { max_atomic_width: Some(64), mcount: "\u{1}__gnu_mcount_nc".into(), llvm_mcount_intrinsic: Some("llvm.arm.gnu.eabi.mcount".into()), + // The default on linux is to have `default_uwtable=true`, but on + // this target we get an "`__aeabi_unwind_cpp_pr0` not defined" + // linker error, so set it to `true` here. + // FIXME(#146996): Remove this override once #146996 has been fixed. + default_uwtable: false, ..base::linux_gnu::opts() }, } |