diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
15 files changed, 642 insertions, 600 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 9e834b83df4..6e3a4cae2f6 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -362,9 +362,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // currently use this mode so we have to allow it -- but we absolutely // shouldn't let any more targets do that. // (Also see <https://github.com/rust-lang/rust/issues/115666>.) + // + // The unstable abi `PtxKernel` also uses Direct for now. + // It needs to switch to something else before stabilization can happen. + // (See issue: https://github.com/rust-lang/rust/issues/117271) assert!( - matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64"), - "`PassMode::Direct` for aggregates only allowed on wasm targets\nProblematic type: {:#?}", + matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64") + || self.conv == Conv::PtxKernel, + "`PassMode::Direct` for aggregates only allowed on wasm and `extern \"ptx-kernel\"` fns\nProblematic type: {:#?}", arg.layout, ); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c778a6e017f..9d5204034de 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -259,9 +259,17 @@ pub fn target_machine_factory( }; let debuginfo_compression = SmallCStr::new(&debuginfo_compression); + let should_prefer_remapped_for_split_debuginfo_paths = + sess.should_prefer_remapped_for_split_debuginfo_paths(); + Arc::new(move |config: TargetMachineFactoryConfig| { let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { - let path = path_mapping.map_prefix(path.unwrap_or_default()).0; + let path = path.unwrap_or_default(); + let path = if should_prefer_remapped_for_split_debuginfo_paths { + path_mapping.map_prefix(path).0 + } else { + path.into() + }; CString::new(path.to_str().unwrap()).unwrap() }; diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 763186a58bf..7ad2d03a5ed 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::coverage::{CounterId, ExpressionId, Operand}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -43,11 +43,11 @@ impl Counter { Self { kind: CounterKind::Expression, id: expression_id.as_u32() } } - pub(crate) fn from_operand(operand: Operand) -> Self { - match operand { - Operand::Zero => Self::ZERO, - Operand::Counter(id) => Self::counter_value_reference(id), - Operand::Expression(id) => Self::expression(id), + pub(crate) fn from_term(term: CovTerm) -> Self { + match term { + CovTerm::Zero => Self::ZERO, + CovTerm::Counter(id) => Self::counter_value_reference(id), + CovTerm::Expression(id) => Self::expression(id), } } } @@ -73,17 +73,6 @@ pub struct CounterExpression { pub rhs: Counter, } -impl CounterExpression { - /// The dummy expression `(0 - 0)` has a representation of all zeroes, - /// making it marginally more efficient to initialize than `(0 + 0)`. - pub(crate) const DUMMY: Self = - Self { lhs: Counter::ZERO, kind: ExprKind::Subtract, rhs: Counter::ZERO }; - - pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { - Self { kind, lhs, rhs } - } -} - /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. /// /// Must match the layout of `LLVMRustCounterMappingRegionKind`. diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 55f43aa5341..93a8a4b1d5e 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -1,294 +1,260 @@ use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; +use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxIndexSet; -use rustc_index::IndexVec; -use rustc_middle::mir::coverage::{CodeRegion, CounterId, ExpressionId, Op, Operand}; +use rustc_index::bit_set::BitSet; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, Op, +}; use rustc_middle::ty::Instance; -use rustc_middle::ty::TyCtxt; - -#[derive(Clone, Debug, PartialEq)] -pub struct Expression { - lhs: Operand, - op: Op, - rhs: Operand, - code_regions: Vec<CodeRegion>, -} +use rustc_span::Symbol; -/// Collects all of the coverage regions associated with (a) injected counters, (b) counter -/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), -/// for a given Function. This struct also stores the `function_source_hash`, -/// computed during instrumentation, and forwarded with counters. -/// -/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap -/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter -/// or expression), but the line or lines in the gap region are not executable (such as lines with -/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count -/// for a gap area is only used as the line execution count if there are no other regions on a -/// line." +/// Holds all of the coverage mapping data associated with a function instance, +/// collected during traversal of `Coverage` statements in the function's MIR. #[derive(Debug)] -pub struct FunctionCoverage<'tcx> { - instance: Instance<'tcx>, - source_hash: u64, +pub struct FunctionCoverageCollector<'tcx> { + /// Coverage info that was attached to this function by the instrumentor. + function_coverage_info: &'tcx FunctionCoverageInfo, is_used: bool, - counters: IndexVec<CounterId, Option<Vec<CodeRegion>>>, - expressions: IndexVec<ExpressionId, Option<Expression>>, - unreachable_regions: Vec<CodeRegion>, + + /// Tracks which counters have been seen, so that we can identify mappings + /// to counters that were optimized out, and set them to zero. + counters_seen: BitSet<CounterId>, + /// Contains all expression IDs that have been seen in an `ExpressionUsed` + /// coverage statement, plus all expression IDs that aren't directly used + /// by any mappings (and therefore do not have expression-used statements). + /// After MIR traversal is finished, we can conclude that any IDs missing + /// from this set must have had their statements deleted by MIR opts. + expressions_seen: BitSet<ExpressionId>, } -impl<'tcx> FunctionCoverage<'tcx> { +impl<'tcx> FunctionCoverageCollector<'tcx> { /// Creates a new set of coverage data for a used (called) function. - pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, true) + pub fn new( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + ) -> Self { + Self::create(instance, function_coverage_info, true) } /// Creates a new set of coverage data for an unused (never called) function. - pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, false) + pub fn unused( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + ) -> Self { + Self::create(instance, function_coverage_info, false) } - fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { - let coverageinfo = tcx.coverageinfo(instance.def); + fn create( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + is_used: bool, + ) -> Self { + let num_counters = function_coverage_info.num_counters; + let num_expressions = function_coverage_info.expressions.len(); debug!( - "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", - instance, coverageinfo, is_used + "FunctionCoverage::create(instance={instance:?}) has \ + num_counters={num_counters}, num_expressions={num_expressions}, is_used={is_used}" ); - Self { - instance, - source_hash: 0, // will be set with the first `add_counter()` - is_used, - counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), - expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), - unreachable_regions: Vec::new(), - } - } - - /// Returns true for a used (called) function, and false for an unused function. - pub fn is_used(&self) -> bool { - self.is_used - } - - /// Sets the function source hash value. If called multiple times for the same function, all - /// calls should have the same hash value. - pub fn set_function_source_hash(&mut self, source_hash: u64) { - if self.source_hash == 0 { - self.source_hash = source_hash; - } else { - debug_assert_eq!(source_hash, self.source_hash); - } - } - /// Adds code regions to be counted by an injected counter intrinsic. - #[instrument(level = "debug", skip(self))] - pub(crate) fn add_counter(&mut self, id: CounterId, code_regions: &[CodeRegion]) { - if code_regions.is_empty() { - return; + // Create a filled set of expression IDs, so that expressions not + // directly used by mappings will be treated as "seen". + // (If they end up being unused, LLVM will delete them for us.) + let mut expressions_seen = BitSet::new_filled(num_expressions); + // For each expression ID that is directly used by one or more mappings, + // mark it as not-yet-seen. This indicates that we expect to see a + // corresponding `ExpressionUsed` statement during MIR traversal. + for Mapping { term, .. } in &function_coverage_info.mappings { + if let &CovTerm::Expression(id) = term { + expressions_seen.remove(id); + } } - let slot = &mut self.counters[id]; - match slot { - None => *slot = Some(code_regions.to_owned()), - // If this counter ID slot has already been filled, it should - // contain identical information. - Some(ref previous_regions) => assert_eq!( - previous_regions, code_regions, - "add_counter: code regions for id changed" - ), + Self { + function_coverage_info, + is_used, + counters_seen: BitSet::new_empty(num_counters), + expressions_seen, } } - /// Adds information about a coverage expression, along with zero or more - /// code regions mapped to that expression. - /// - /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other - /// expressions. These are tracked as separate variants of `Operand`, so there is no ambiguity - /// between operands that are counter IDs and operands that are expression IDs. + /// Marks a counter ID as having been seen in a counter-increment statement. #[instrument(level = "debug", skip(self))] - pub(crate) fn add_counter_expression( - &mut self, - expression_id: ExpressionId, - lhs: Operand, - op: Op, - rhs: Operand, - code_regions: &[CodeRegion], - ) { - debug_assert!( - expression_id.as_usize() < self.expressions.len(), - "expression_id {} is out of range for expressions.len() = {} - for {:?}", - expression_id.as_usize(), - self.expressions.len(), - self, - ); - - let expression = Expression { lhs, op, rhs, code_regions: code_regions.to_owned() }; - let slot = &mut self.expressions[expression_id]; - match slot { - None => *slot = Some(expression), - // If this expression ID slot has already been filled, it should - // contain identical information. - Some(ref previous_expression) => assert_eq!( - previous_expression, &expression, - "add_counter_expression: expression for id changed" - ), - } + pub(crate) fn mark_counter_id_seen(&mut self, id: CounterId) { + self.counters_seen.insert(id); } - /// Adds regions that will be marked as "unreachable", with a constant "zero counter". + /// Marks an expression ID as having been seen in an expression-used statement. #[instrument(level = "debug", skip(self))] - pub(crate) fn add_unreachable_regions(&mut self, code_regions: &[CodeRegion]) { - assert!(!code_regions.is_empty(), "unreachable regions always have code regions"); - self.unreachable_regions.extend_from_slice(code_regions); + pub(crate) fn mark_expression_id_seen(&mut self, id: ExpressionId) { + self.expressions_seen.insert(id); } - /// Perform some simplifications to make the final coverage mappings - /// slightly smaller. + /// Identify expressions that will always have a value of zero, and note + /// their IDs in [`ZeroExpressions`]. Mappings that refer to a zero expression + /// can instead become mappings to a constant zero value. /// /// This method mainly exists to preserve the simplifications that were /// already being performed by the Rust-side expression renumbering, so that /// the resulting coverage mappings don't get worse. - pub(crate) fn simplify_expressions(&mut self) { + fn identify_zero_expressions(&self) -> ZeroExpressions { // The set of expressions that either were optimized out entirely, or // have zero as both of their operands, and will therefore always have // a value of zero. Other expressions that refer to these as operands - // can have those operands replaced with `Operand::Zero`. + // can have those operands replaced with `CovTerm::Zero`. let mut zero_expressions = FxIndexSet::default(); - // For each expression, perform simplifications based on lower-numbered - // expressions, and then update the set of always-zero expressions if - // necessary. + // Simplify a copy of each expression based on lower-numbered expressions, + // and then update the set of always-zero expressions if necessary. // (By construction, expressions can only refer to other expressions - // that have lower IDs, so one simplification pass is sufficient.) - for (id, maybe_expression) in self.expressions.iter_enumerated_mut() { - let Some(expression) = maybe_expression else { - // If an expression is missing, it must have been optimized away, + // that have lower IDs, so one pass is sufficient.) + for (id, expression) in self.function_coverage_info.expressions.iter_enumerated() { + if !self.expressions_seen.contains(id) { + // If an expression was not seen, it must have been optimized away, // so any operand that refers to it can be replaced with zero. zero_expressions.insert(id); continue; + } + + // We don't need to simplify the actual expression data in the + // expressions list; we can just simplify a temporary copy and then + // use that to update the set of always-zero expressions. + let Expression { mut lhs, op, mut rhs } = *expression; + + // If an expression has an operand that is also an expression, the + // operand's ID must be strictly lower. This is what lets us find + // all zero expressions in one pass. + let assert_operand_expression_is_lower = |operand_id: ExpressionId| { + assert!( + operand_id < id, + "Operand {operand_id:?} should be less than {id:?} in {expression:?}", + ) }; // If an operand refers to an expression that is always zero, then - // that operand can be replaced with `Operand::Zero`. - let maybe_set_operand_to_zero = |operand: &mut Operand| match &*operand { - Operand::Expression(id) if zero_expressions.contains(id) => { - *operand = Operand::Zero; + // that operand can be replaced with `CovTerm::Zero`. + let maybe_set_operand_to_zero = |operand: &mut CovTerm| match *operand { + CovTerm::Expression(id) => { + assert_operand_expression_is_lower(id); + if zero_expressions.contains(&id) { + *operand = CovTerm::Zero; + } } _ => (), }; - maybe_set_operand_to_zero(&mut expression.lhs); - maybe_set_operand_to_zero(&mut expression.rhs); + maybe_set_operand_to_zero(&mut lhs); + maybe_set_operand_to_zero(&mut rhs); // Coverage counter values cannot be negative, so if an expression // involves subtraction from zero, assume that its RHS must also be zero. // (Do this after simplifications that could set the LHS to zero.) - if let Expression { lhs: Operand::Zero, op: Op::Subtract, .. } = expression { - expression.rhs = Operand::Zero; + if lhs == CovTerm::Zero && op == Op::Subtract { + rhs = CovTerm::Zero; } // After the above simplifications, if both operands are zero, then // we know that this expression is always zero too. - if let Expression { lhs: Operand::Zero, rhs: Operand::Zero, .. } = expression { + if lhs == CovTerm::Zero && rhs == CovTerm::Zero { zero_expressions.insert(id); } } + + ZeroExpressions(zero_expressions) } - /// Return the source hash, generated from the HIR node structure, and used to indicate whether - /// or not the source code structure changed between different compilations. - pub fn source_hash(&self) -> u64 { - self.source_hash + pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> { + let zero_expressions = self.identify_zero_expressions(); + let FunctionCoverageCollector { function_coverage_info, is_used, counters_seen, .. } = self; + + FunctionCoverage { function_coverage_info, is_used, counters_seen, zero_expressions } } +} - /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their - /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create - /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions( - &self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { - assert!( - self.source_hash != 0 || !self.is_used, - "No counters provided the source_hash for used function: {:?}", - self.instance - ); +pub(crate) struct FunctionCoverage<'tcx> { + function_coverage_info: &'tcx FunctionCoverageInfo, + is_used: bool, - let counter_expressions = self.counter_expressions(); - // Expression IDs are indices into `self.expressions`, and on the LLVM - // side they will be treated as indices into `counter_expressions`, so - // the two vectors should correspond 1:1. - assert_eq!(self.expressions.len(), counter_expressions.len()); + counters_seen: BitSet<CounterId>, + zero_expressions: ZeroExpressions, +} - let counter_regions = self.counter_regions(); - let expression_regions = self.expression_regions(); - let unreachable_regions = self.unreachable_regions(); +impl<'tcx> FunctionCoverage<'tcx> { + /// Returns true for a used (called) function, and false for an unused function. + pub(crate) fn is_used(&self) -> bool { + self.is_used + } - let counter_regions = - counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); - (counter_expressions, counter_regions) + /// Return the source hash, generated from the HIR node structure, and used to indicate whether + /// or not the source code structure changed between different compilations. + pub fn source_hash(&self) -> u64 { + if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } } - fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.counters - .iter_enumerated() - // Filter out counter IDs that we never saw during MIR traversal. - // This can happen if a counter was optimized out by MIR transforms - // (and replaced with `CoverageKind::Unreachable` instead). - .filter_map(|(id, maybe_code_regions)| Some((id, maybe_code_regions.as_ref()?))) - .flat_map(|(id, code_regions)| { - let counter = Counter::counter_value_reference(id); - code_regions.iter().map(move |region| (counter, region)) - }) + /// Returns an iterator over all filenames used by this function's mappings. + pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> { + self.function_coverage_info.mappings.iter().map(|mapping| mapping.code_region.file_name) } /// Convert this function's coverage expression data into a form that can be /// passed through FFI to LLVM. - fn counter_expressions(&self) -> Vec<CounterExpression> { + pub(crate) fn counter_expressions( + &self, + ) -> impl Iterator<Item = CounterExpression> + ExactSizeIterator + Captures<'_> { // We know that LLVM will optimize out any unused expressions before // producing the final coverage map, so there's no need to do the same // thing on the Rust side unless we're confident we can do much better. // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.) - self.expressions - .iter() - .map(|expression| match expression { - None => { - // This expression ID was allocated, but we never saw the - // actual expression, so it must have been optimized out. - // Replace it with a dummy expression, and let LLVM take - // care of omitting it from the expression list. - CounterExpression::DUMMY - } - &Some(Expression { lhs, op, rhs, .. }) => { - // Convert the operands and operator as normal. - CounterExpression::new( - Counter::from_operand(lhs), - match op { - Op::Add => ExprKind::Add, - Op::Subtract => ExprKind::Subtract, - }, - Counter::from_operand(rhs), - ) - } - }) - .collect::<Vec<_>>() + let counter_from_operand = |operand: CovTerm| match operand { + CovTerm::Expression(id) if self.zero_expressions.contains(id) => Counter::ZERO, + _ => Counter::from_term(operand), + }; + + self.function_coverage_info.expressions.iter().map(move |&Expression { lhs, op, rhs }| { + CounterExpression { + lhs: counter_from_operand(lhs), + kind: match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + rhs: counter_from_operand(rhs), + } + }) } - fn expression_regions(&self) -> Vec<(Counter, &CodeRegion)> { - // Find all of the expression IDs that weren't optimized out AND have - // one or more attached code regions, and return the corresponding - // mappings as counter/region pairs. - self.expressions - .iter_enumerated() - .filter_map(|(id, maybe_expression)| { - let code_regions = &maybe_expression.as_ref()?.code_regions; - Some((id, code_regions)) - }) - .flat_map(|(id, code_regions)| { - let counter = Counter::expression(id); - code_regions.iter().map(move |code_region| (counter, code_region)) - }) - .collect::<Vec<_>>() + /// Converts this function's coverage mappings into an intermediate form + /// that will be used by `mapgen` when preparing for FFI. + pub(crate) fn counter_regions( + &self, + ) -> impl Iterator<Item = (Counter, &CodeRegion)> + ExactSizeIterator { + // Historically, mappings were stored directly in counter/expression + // statements in MIR, and MIR optimizations would sometimes remove them. + // That's mostly no longer true, so now we detect cases where that would + // have happened, and zero out the corresponding mappings here instead. + let counter_for_term = move |term: CovTerm| { + let force_to_zero = match term { + CovTerm::Counter(id) => !self.counters_seen.contains(id), + CovTerm::Expression(id) => self.zero_expressions.contains(id), + CovTerm::Zero => false, + }; + if force_to_zero { Counter::ZERO } else { Counter::from_term(term) } + }; + + self.function_coverage_info.mappings.iter().map(move |mapping| { + let &Mapping { term, ref code_region } = mapping; + let counter = counter_for_term(term); + (counter, code_region) + }) } +} + +/// Set of expression IDs that are known to always evaluate to zero. +/// Any mapping or expression operand that refers to these expressions can have +/// that reference replaced with a constant zero value. +struct ZeroExpressions(FxIndexSet<ExpressionId>); - fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.unreachable_regions.iter().map(|region| (Counter::ZERO, region)) +impl ZeroExpressions { + fn contains(&self, id: ExpressionId) -> bool { + self.0.contains(&id) } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index d4e77525698..274e0aeaaba 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,18 +1,20 @@ use crate::common::CodegenCx; use crate::coverageinfo; use crate::coverageinfo::ffi::CounterMappingRegion; -use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::llvm; -use rustc_codegen_ssa::traits::ConstMethods; -use rustc_data_structures::fx::FxIndexSet; +use itertools::Itertools as _; +use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods}; +use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_index::IndexVec; use rustc_middle::bug; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir; use rustc_middle::mir::coverage::CodeRegion; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::def_id::DefIdSet; use rustc_span::Symbol; /// Generates and exports the Coverage Map. @@ -56,21 +58,40 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { return; } - let mut global_file_table = GlobalFileTable::new(tcx); + let function_coverage_entries = function_coverage_map + .into_iter() + .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) + .collect::<Vec<_>>(); + + let all_file_names = + function_coverage_entries.iter().flat_map(|(_, fn_cov)| fn_cov.all_file_names()); + let global_file_table = GlobalFileTable::new(all_file_names); + + // Encode all filenames referenced by coverage mappings in this CGU. + let filenames_buffer = global_file_table.make_filenames_buffer(tcx); + + let filenames_size = filenames_buffer.len(); + let filenames_val = cx.const_bytes(&filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); + + // Generate the coverage map header, which contains the filenames used by + // this CGU's coverage mappings, and store it in a well-known global. + let cov_data_val = generate_coverage_map(cx, version, filenames_size, filenames_val); + coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + + let mut unused_function_names = Vec::new(); + let covfun_section_name = coverageinfo::covfun_section_name(cx); // Encode coverage mappings and generate function records - let mut function_data = Vec::new(); - for (instance, mut function_coverage) in function_coverage_map { + for (instance, function_coverage) in function_coverage_entries { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); - function_coverage.simplify_expressions(); - let function_coverage = function_coverage; let mangled_function_name = tcx.symbol_name(instance).name; let source_hash = function_coverage.source_hash(); let is_used = function_coverage.is_used(); let coverage_mapping_buffer = - encode_mappings_for_function(&mut global_file_table, &function_coverage); + encode_mappings_for_function(&global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { if function_coverage.is_used() { @@ -84,21 +105,10 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { } } - function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); - } - - // Encode all filenames referenced by counters/expressions in this module - let filenames_buffer = global_file_table.into_filenames_buffer(); - - let filenames_size = filenames_buffer.len(); - let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); - - // Generate the LLVM IR representation of the coverage map and store it in a well-known global - let cov_data_val = generate_coverage_map(cx, version, filenames_size, filenames_val); + if !is_used { + unused_function_names.push(mangled_function_name); + } - let covfun_section_name = coverageinfo::covfun_section_name(cx); - for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { save_function_record( cx, &covfun_section_name, @@ -110,90 +120,143 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { ); } - // Save the coverage data value to LLVM IR - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + // For unused functions, we need to take their mangled names and store them + // in a specially-named global array. LLVM's `InstrProfiling` pass will + // detect this global and include those names in its `__llvm_prf_names` + // section. (See `llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp`.) + if !unused_function_names.is_empty() { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let name_globals = unused_function_names + .into_iter() + .map(|mangled_function_name| cx.const_str(mangled_function_name).0) + .collect::<Vec<_>>(); + let initializer = cx.const_array(cx.type_ptr(), &name_globals); + + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), "__llvm_coverage_names"); + llvm::set_global_constant(array, true); + llvm::set_linkage(array, llvm::Linkage::InternalLinkage); + llvm::set_initializer(array, initializer); + } } +/// Maps "global" (per-CGU) file ID numbers to their underlying filenames. struct GlobalFileTable { - global_file_table: FxIndexSet<Symbol>, + /// This "raw" table doesn't include the working dir, so a filename's + /// global ID is its index in this set **plus one**. + raw_file_table: FxIndexSet<Symbol>, } impl GlobalFileTable { - fn new(tcx: TyCtxt<'_>) -> Self { - let mut global_file_table = FxIndexSet::default(); + fn new(all_file_names: impl IntoIterator<Item = Symbol>) -> Self { + // Collect all of the filenames into a set. Filenames usually come in + // contiguous runs, so we can dedup adjacent ones to save work. + let mut raw_file_table = all_file_names.into_iter().dedup().collect::<FxIndexSet<Symbol>>(); + + // Sort the file table by its actual string values, not the arbitrary + // ordering of its symbols. + raw_file_table.sort_unstable_by(|a, b| a.as_str().cmp(b.as_str())); + + Self { raw_file_table } + } + + fn global_file_id_for_file_name(&self, file_name: Symbol) -> u32 { + let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| { + bug!("file name not found in prepared global file table: {file_name}"); + }); + // The raw file table doesn't include an entry for the working dir + // (which has ID 0), so add 1 to get the correct ID. + (raw_id + 1) as u32 + } + + fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) // requires setting the first filename to the compilation directory. // Since rustc generates coverage maps with relative paths, the // compilation directory can be combined with the relative paths // to get absolute paths, if needed. - let working_dir = Symbol::intern( - &tcx.sess.opts.working_dir.remapped_path_if_available().to_string_lossy(), - ); - global_file_table.insert(working_dir); - Self { global_file_table } - } - - fn global_file_id_for_file_name(&mut self, file_name: Symbol) -> u32 { - let (global_file_id, _) = self.global_file_table.insert_full(file_name); - global_file_id as u32 - } - - fn into_filenames_buffer(self) -> Vec<u8> { - // This method takes `self` so that the caller can't accidentally - // modify the original file table after encoding it into a buffer. + use rustc_session::RemapFileNameExt; + let working_dir: &str = &tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy(); llvm::build_byte_buffer(|buffer| { coverageinfo::write_filenames_section_to_buffer( - self.global_file_table.iter().map(Symbol::as_str), + // Insert the working dir at index 0, before the other filenames. + std::iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)), buffer, ); }) } } +rustc_index::newtype_index! { + // Tell the newtype macro to not generate `Encode`/`Decode` impls. + #[custom_encodable] + struct LocalFileId {} +} + +/// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) +/// file IDs. +#[derive(Default)] +struct VirtualFileMapping { + local_to_global: IndexVec<LocalFileId, u32>, + global_to_local: FxIndexMap<u32, LocalFileId>, +} + +impl VirtualFileMapping { + fn local_id_for_global(&mut self, global_file_id: u32) -> LocalFileId { + *self + .global_to_local + .entry(global_file_id) + .or_insert_with(|| self.local_to_global.push(global_file_id)) + } + + fn into_vec(self) -> Vec<u32> { + self.local_to_global.raw + } +} + /// Using the expressions and counter regions collected for a single function, /// generate the variable-sized payload of its corresponding `__llvm_covfun` /// entry. The payload is returned as a vector of bytes. /// /// Newly-encountered filenames will be added to the global file table. fn encode_mappings_for_function( - global_file_table: &mut GlobalFileTable, + global_file_table: &GlobalFileTable, function_coverage: &FunctionCoverage<'_>, ) -> Vec<u8> { - let (expressions, counter_regions) = function_coverage.get_expressions_and_counter_regions(); - - let mut counter_regions = counter_regions.collect::<Vec<_>>(); + let counter_regions = function_coverage.counter_regions(); if counter_regions.is_empty() { return Vec::new(); } - let mut virtual_file_mapping = IndexVec::<u32, u32>::new(); + let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); + + let mut virtual_file_mapping = VirtualFileMapping::default(); let mut mapping_regions = Vec::with_capacity(counter_regions.len()); - // Sort the list of (counter, region) mapping pairs by region, so that they - // can be grouped by filename. Prepare file IDs for each filename, and - // prepare the mapping data so that we can pass it through FFI to LLVM. - counter_regions.sort_by_key(|(_counter, region)| *region); - for counter_regions_for_file in - counter_regions.group_by(|(_, a), (_, b)| a.file_name == b.file_name) + // Group mappings into runs with the same filename, preserving the order + // yielded by `FunctionCoverage`. + // Prepare file IDs for each filename, and prepare the mapping data so that + // we can pass it through FFI to LLVM. + for (file_name, counter_regions_for_file) in + &counter_regions.group_by(|(_counter, region)| region.file_name) { - // Look up (or allocate) the global file ID for this filename. - let file_name = counter_regions_for_file[0].1.file_name; + // Look up the global file ID for this filename. let global_file_id = global_file_table.global_file_id_for_file_name(file_name); // Associate that global file ID with a local file ID for this function. - let local_file_id: u32 = virtual_file_mapping.push(global_file_id); - debug!(" file id: local {local_file_id} => global {global_file_id} = '{file_name:?}'"); + let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => global {global_file_id} = '{file_name:?}'"); // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - for &(counter, region) in counter_regions_for_file { + for (counter, region) in counter_regions_for_file { let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = *region; debug!("Adding counter {counter:?} to map for {region:?}"); mapping_regions.push(CounterMappingRegion::code_region( counter, - local_file_id, + local_file_id.as_u32(), start_line, start_col, end_line, @@ -205,7 +268,7 @@ fn encode_mappings_for_function( // Encode the function's coverage mappings into a buffer. llvm::build_byte_buffer(|buffer| { coverageinfo::write_mapping_to_buffer( - virtual_file_mapping.raw, + virtual_file_mapping.into_vec(), expressions, mapping_regions, buffer, @@ -289,13 +352,12 @@ fn save_function_record( /// `-Clink-dead-code` will not generate code for unused generic functions.) /// /// We can find the unused functions (including generic functions) by the set difference of all MIR -/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query -/// `codegened_and_inlined_items`). +/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`codegenned_and_inlined_items`). /// -/// These unused functions are then codegen'd in one of the CGUs which is marked as the -/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating -/// code regions for the same function more than once which can lead to linker errors regarding -/// duplicate symbols. +/// These unused functions don't need to be codegenned, but we do need to add them to the function +/// coverage map (in a single designated CGU) so that we still emit coverage mappings for them. +/// We also end up adding their symbol names to a special global array that LLVM will include in +/// its embedded coverage data. fn add_unused_functions(cx: &CodegenCx<'_, '_>) { assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); @@ -315,7 +377,7 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { // generic functions from consideration as well. if !matches!( kind, - DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator + DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Coroutine ) { return None; } @@ -326,21 +388,80 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { }) .collect(); - let codegenned_def_ids = tcx.codegened_and_inlined_items(()); + let codegenned_def_ids = codegenned_and_inlined_items(tcx); - for non_codegenned_def_id in - eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) - { - let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - - // If a function is marked `#[coverage(off)]`, then skip generating a - // dead code stub for it. - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { - debug!("skipping unused fn marked #[coverage(off)]: {:?}", non_codegenned_def_id); + // For each `DefId` that should have coverage instrumentation but wasn't + // codegenned, add it to the function coverage map as an unused function. + for def_id in eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) { + // Skip any function that didn't have coverage data added to it by the + // coverage instrumentor. + let body = tcx.instance_mir(ty::InstanceDef::Item(def_id)); + let Some(function_coverage_info) = body.function_coverage_info.as_deref() else { continue; + }; + + debug!("generating unused fn: {def_id:?}"); + let instance = declare_unused_fn(tcx, def_id); + add_unused_function_coverage(cx, instance, function_coverage_info); + } +} + +/// All items participating in code generation together with (instrumented) +/// items inlined into them. +fn codegenned_and_inlined_items(tcx: TyCtxt<'_>) -> DefIdSet { + let (items, cgus) = tcx.collect_and_partition_mono_items(()); + let mut visited = DefIdSet::default(); + let mut result = items.clone(); + + for cgu in cgus { + for item in cgu.items().keys() { + if let mir::mono::MonoItem::Fn(ref instance) = item { + let did = instance.def_id(); + if !visited.insert(did) { + continue; + } + let body = tcx.instance_mir(instance.def); + for block in body.basic_blocks.iter() { + for statement in &block.statements { + let mir::StatementKind::Coverage(_) = statement.kind else { continue }; + let scope = statement.source_info.scope; + if let Some(inlined) = scope.inlined_instance(&body.source_scopes) { + result.insert(inlined.def_id()); + } + } + } + } } + } - debug!("generating unused fn: {:?}", non_codegenned_def_id); - cx.define_unused_fn(non_codegenned_def_id); + result +} + +fn declare_unused_fn<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> ty::Instance<'tcx> { + ty::Instance::new( + def_id, + ty::GenericArgs::for_item(tcx, def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ) +} + +fn add_unused_function_coverage<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + instance: ty::Instance<'tcx>, + function_coverage_info: &'tcx mir::coverage::FunctionCoverageInfo, +) { + // An unused function's mappings will automatically be rewritten to map to + // zero, because none of its counters/expressions are marked as seen. + let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); + + if let Some(coverage_context) = cx.coverage_context() { + coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); + } else { + bug!("Could not get the `coverage_context`"); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index dd2ce9b525b..7d69756181a 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,10 +1,9 @@ use crate::llvm; -use crate::abi::Abi; use crate::builder::Builder; use crate::common::CodegenCx; use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; -use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::map_data::FunctionCoverageCollector; use libc::c_uint; use rustc_codegen_ssa::traits::{ @@ -12,17 +11,12 @@ use rustc_codegen_ssa::traits::{ StaticMethods, }; use rustc_data_structures::fx::FxHashMap; -use rustc_hir as hir; -use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{CounterId, CoverageKind}; +use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::Coverage; -use rustc_middle::ty; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::GenericArgs; +use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; -use rustc_middle::ty::Ty; use std::cell::RefCell; @@ -30,14 +24,13 @@ pub(crate) mod ffi; pub(crate) mod map_data; pub mod mapgen; -const UNUSED_FUNCTION_COUNTER_ID: CounterId = CounterId::START; - const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, + pub(crate) function_coverage_map: + RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -49,7 +42,9 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } - pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { + pub fn take_function_coverage_map( + &self, + ) -> FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { self.function_coverage_map.replace(FxHashMap::default()) } } @@ -76,56 +71,56 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { bug!("Could not get the `coverage_context`"); } } - - /// Functions with MIR-based coverage are normally codegenned _only_ if - /// called. LLVM coverage tools typically expect every function to be - /// defined (even if unused), with at least one call to LLVM intrinsic - /// `instrprof.increment`. - /// - /// Codegen a small function that will never be called, with one counter - /// that will never be incremented. - /// - /// For used/called functions, the coverageinfo was already added to the - /// `function_coverage_map` (keyed by function `Instance`) during codegen. - /// But in this case, since the unused function was _not_ previously - /// codegenned, collect the coverage `CodeRegion`s from the MIR and add - /// them. Since the function is never called, all of its `CodeRegion`s can be - /// added as `unreachable_region`s. - fn define_unused_fn(&self, def_id: DefId) { - let instance = declare_unused_fn(self, def_id); - codegen_unused_fn_and_counter(self, instance); - add_unused_function_coverage(self, instance, def_id); - } } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { + // Our caller should have already taken care of inlining subtleties, + // so we can assume that counter/expression IDs in this coverage + // statement are meaningful for the given instance. + // + // (Either the statement was not inlined and directly belongs to this + // instance, or it was inlined *from* this instance.) + let bx = self; + let Some(function_coverage_info) = + bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() + else { + debug!("function has a coverage statement but no coverage info"); + return; + }; + let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map .entry(instance) - .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); + .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); - let Coverage { kind, code_regions } = coverage; + let Coverage { kind } = coverage; match *kind { - CoverageKind::Counter { function_source_hash, id } => { - debug!( - "ensuring function source hash is set for instance={:?}; function_source_hash={}", - instance, function_source_hash, - ); - func_coverage.set_function_source_hash(function_source_hash); - func_coverage.add_counter(id, code_regions); + CoverageKind::CounterIncrement { id } => { + func_coverage.mark_counter_id_seen(id); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, // as that needs an exclusive borrow. drop(coverage_map); - let coverageinfo = bx.tcx().coverageinfo(instance.def); + // The number of counters passed to `llvm.instrprof.increment` might + // be smaller than the number originally inserted by the instrumentor, + // if some high-numbered counters were removed by MIR optimizations. + // If so, LLVM's profiler runtime will use fewer physical counters. + let num_counters = + bx.tcx().coverage_ids_info(instance.def).max_counter_id.as_u32() + 1; + assert!( + num_counters as usize <= function_coverage_info.num_counters, + "num_counters disagreement: query says {num_counters} but function info only has {}", + function_coverage_info.num_counters + ); let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let num_counters = bx.const_u32(num_counters); let index = bx.const_u32(id.as_u32()); debug!( "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", @@ -133,90 +128,13 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::Expression { id, lhs, op, rhs } => { - func_coverage.add_counter_expression(id, lhs, op, rhs, code_regions); - } - CoverageKind::Unreachable => { - func_coverage.add_unreachable_regions(code_regions); + CoverageKind::ExpressionUsed { id } => { + func_coverage.mark_expression_id_seen(id); } } } } -fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { - let tcx = cx.tcx; - - let instance = Instance::new( - def_id, - GenericArgs::for_item(tcx, def_id, |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }), - ); - - let llfn = cx.declare_fn( - tcx.symbol_name(instance).name, - cx.fn_abi_of_fn_ptr( - ty::Binder::dummy(tcx.mk_fn_sig( - [Ty::new_unit(tcx)], - Ty::new_unit(tcx), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - )), - ty::List::empty(), - ), - None, - ); - - llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); - llvm::set_visibility(llfn, llvm::Visibility::Default); - - assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); - - instance -} - -fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { - let llfn = cx.get_fn(instance); - let llbb = Builder::append_block(cx, llfn, "unused_function"); - let mut bx = Builder::build(cx, llbb); - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(0); - let num_counters = bx.const_u32(1); - let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, - index={:?}) for unused function: {:?}", - fn_name, hash, num_counters, index, instance - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); - bx.ret_void(); -} - -fn add_unused_function_coverage<'tcx>( - cx: &CodegenCx<'_, 'tcx>, - instance: Instance<'tcx>, - def_id: DefId, -) { - let tcx = cx.tcx; - - let mut function_coverage = FunctionCoverage::unused(tcx, instance); - for &code_region in tcx.covered_code_regions(def_id) { - let code_region = std::slice::from_ref(code_region); - function_coverage.add_unreachable_regions(code_region); - } - - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } -} - /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 11874898a5a..865bf01c8c1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -460,7 +460,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), - ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), + ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), AdtKind::Union => build_union_type_di_node(cx, unique_type_id), @@ -547,48 +547,77 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> ) -> &'ll DIFile { debug!(?source_file.name); + use rustc_session::RemapFileNameExt; let (directory, file_name) = match &source_file.name { FileName::Real(filename) => { let working_directory = &cx.sess().opts.working_dir; debug!(?working_directory); - let filename = cx - .sess() - .source_map() - .path_mapping() - .to_embeddable_absolute_path(filename.clone(), working_directory); - - // Construct the absolute path of the file - let abs_path = filename.remapped_path_if_available(); - debug!(?abs_path); - - if let Ok(rel_path) = - abs_path.strip_prefix(working_directory.remapped_path_if_available()) - { - // If the compiler's working directory (which also is the DW_AT_comp_dir of - // the compilation unit) is a prefix of the path we are about to emit, then - // only emit the part relative to the working directory. - // Because of path remapping we sometimes see strange things here: `abs_path` - // might actually look like a relative path - // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without - // taking the working directory into account, downstream tooling will - // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, - // which makes no sense. Usually in such cases the working directory will also - // be remapped to `<crate-name-and-version>` or some other prefix of the path - // we are remapping, so we end up with - // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. - // By moving the working directory portion into the `directory` part of the - // DIFile, we allow LLVM to emit just the relative path for DWARF, while - // still emitting the correct absolute path for CodeView. - ( - working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), - rel_path.to_string_lossy().into_owned(), - ) + if cx.sess().should_prefer_remapped_for_codegen() { + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `<crate-name-and-version>` or some other prefix of the path + // we are remapping, so we end up with + // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } } else { - ("".into(), abs_path.to_string_lossy().into_owned()) + let working_directory = working_directory.local_path_if_available(); + let filename = filename.local_path_if_available(); + + debug!(?working_directory, ?filename); + + let abs_path: Cow<'_, Path> = if filename.is_absolute() { + filename.into() + } else { + let mut p = PathBuf::new(); + p.push(working_directory); + p.push(filename); + p.into() + }; + + if let Ok(rel_path) = abs_path.strip_prefix(working_directory) { + ( + working_directory.to_string_lossy().into(), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } } } - other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + other => { + debug!(?other); + ("".into(), other.for_codegen(cx.sess()).to_string_lossy().into_owned()) + } }; let hash_kind = match source_file.src_hash.kind { @@ -822,8 +851,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. let producer = format!("clang LLVM ({rustc_producer})"); + use rustc_session::RemapFileNameExt; let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); - let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); + let work_dir = tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy(); let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { @@ -834,7 +864,13 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( Some(codegen_unit_name), ) // We get a path relative to the working directory from split_dwarf_path - .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) + .map(|f| { + if tcx.sess.should_prefer_remapped_for_split_debuginfo_paths() { + tcx.sess.source_map().path_mapping().map_prefix(f).0 + } else { + f.into() + } + }) } else { None } @@ -990,20 +1026,20 @@ fn build_struct_type_di_node<'ll, 'tcx>( // Tuples //=----------------------------------------------------------------------------- -/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. -/// For a generator, this will handle upvars shared by all states. +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or coroutine. +/// For a coroutine, this will handle upvars shared by all states. fn build_upvar_field_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - closure_or_generator_ty: Ty<'tcx>, - closure_or_generator_di_node: &'ll DIType, + closure_or_coroutine_ty: Ty<'tcx>, + closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { - let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { - ty::Generator(def_id, args, _) => (def_id, args.as_generator().prefix_tys()), + let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { + ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), _ => { bug!( - "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", - closure_or_generator_ty + "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", + closure_or_coroutine_ty ) } }; @@ -1013,7 +1049,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( ); let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); - let layout = cx.layout_of(closure_or_generator_ty); + let layout = cx.layout_of(closure_or_coroutine_ty); up_var_tys .into_iter() @@ -1022,7 +1058,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( .map(|(index, (up_var_ty, capture_name))| { build_field_di_node( cx, - closure_or_generator_di_node, + closure_or_coroutine_di_node, capture_name.as_str(), cx.size_and_align_of(up_var_ty), layout.fields.offset(index), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 88040557a9b..ca7bfbeac25 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -12,7 +12,7 @@ use rustc_middle::{ ty::{ self, layout::{LayoutOf, TyAndLayout}, - AdtDef, GeneratorArgs, Ty, + AdtDef, CoroutineArgs, Ty, }, }; use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; @@ -268,18 +268,18 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ) } -/// A generator debuginfo node looks the same as a that of an enum type. +/// A coroutine debuginfo node looks the same as a that of an enum type. /// /// See [build_enum_type_di_node] for more information. -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let generator_type_and_layout = cx.layout_of(generator_type); - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + let coroutine_type = unique_type_id.expect_ty(); + let coroutine_type_and_layout = cx.layout_of(coroutine_type); + let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + debug_assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); type_map::build_type_with_children( cx, @@ -287,24 +287,24 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( cx, type_map::Stub::Union, unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), + &coroutine_type_name, + size_and_align_of(coroutine_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, ), - |cx, generator_type_di_node| match generator_type_and_layout.variants { + |cx, coroutine_type_di_node| match coroutine_type_and_layout.variants { Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { - build_union_fields_for_direct_tag_generator( + build_union_fields_for_direct_tag_coroutine( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, ) } Variants::Single { .. } | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout + "Encountered coroutine with non-direct-tag layout: {:?}", + coroutine_type_and_layout ) } }, @@ -428,7 +428,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( }) .collect(); - build_union_fields_for_direct_tag_enum_or_generator( + build_union_fields_for_direct_tag_enum_or_coroutine( cx, enum_type_and_layout, enum_type_di_node, @@ -469,8 +469,8 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_or_generator_type_and_layout: TyAndLayout<'tcx>, - enum_or_generator_type_di_node: &'ll DIType, + enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>, + enum_or_coroutine_type_di_node: &'ll DIType, variant_index: VariantIdx, untagged_variant_index: Option<VariantIdx>, variant_struct_type_di_node: &'ll DIType, @@ -486,13 +486,13 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( Stub::Struct, UniqueTypeId::for_enum_variant_struct_type_wrapper( cx.tcx, - enum_or_generator_type_and_layout.ty, + enum_or_coroutine_type_and_layout.ty, variant_index, ), &variant_struct_wrapper_type_name(variant_index), // NOTE: We use size and align of enum_type, not from variant_layout: - size_and_align_of(enum_or_generator_type_and_layout), - Some(enum_or_generator_type_di_node), + size_and_align_of(enum_or_coroutine_type_and_layout), + Some(enum_or_coroutine_type_di_node), DIFlags::FlagZero, ), |cx, wrapper_struct_type_di_node| { @@ -535,7 +535,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( cx, wrapper_struct_type_di_node, "value", - size_and_align_of(enum_or_generator_type_and_layout), + size_and_align_of(enum_or_coroutine_type_and_layout), Size::ZERO, DIFlags::FlagZero, variant_struct_type_di_node, @@ -662,40 +662,40 @@ fn split_128(value: u128) -> Split128 { Split128 { hi: (value >> 64) as u64, lo: value as u64 } } -fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( +fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, + coroutine_type_and_layout: TyAndLayout<'tcx>, + coroutine_type_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = - generator_type_and_layout.variants + coroutine_type_and_layout.variants else { bug!("This function only supports layouts with directly encoded tags.") }; - let (generator_def_id, generator_args) = match generator_type_and_layout.ty.kind() { - &ty::Generator(def_id, args, _) => (def_id, args.as_generator()), + let (coroutine_def_id, coroutine_args) = match coroutine_type_and_layout.ty.kind() { + &ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine()), _ => unreachable!(), }; - let generator_layout = cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); + let coroutine_layout = cx.tcx.optimized_mir(coroutine_def_id).coroutine_layout().unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); - let variant_range = generator_args.variant_range(generator_def_id, cx.tcx); + let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); + let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); - let tag_base_type = tag_base_type(cx, generator_type_and_layout); + let tag_base_type = tag_base_type(cx, coroutine_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, - generator_type_di_node, + coroutine_type_di_node, variant_range .clone() - .map(|variant_index| (variant_index, GeneratorArgs::variant_name(variant_index))), + .map(|variant_index| (variant_index, CoroutineArgs::variant_name(variant_index))), ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { - let discriminants_iter = generator_args.discriminants(generator_def_id, cx.tcx); + let discriminants_iter = coroutine_args.discriminants(coroutine_def_id, cx.tcx); let mut discriminants: IndexVec<VariantIdx, DiscrResult> = IndexVec::with_capacity(variant_count); for (variant_index, discr) in discriminants_iter { @@ -709,16 +709,16 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range .map(|variant_index| { - let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( + let variant_struct_type_di_node = super::build_coroutine_variant_struct_type_di_node( cx, variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_layout, &common_upvar_names, ); - let span = generator_layout.variant_source_info[variant_index].span; + let span = coroutine_layout.variant_source_info[variant_index].span; let source_info = if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); Some((file_metadata(cx, &loc.file), loc.line as c_uint)) @@ -735,10 +735,10 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( }) .collect(); - build_union_fields_for_direct_tag_enum_or_generator( + build_union_fields_for_direct_tag_enum_or_coroutine( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, &variant_field_infos[..], variant_names_type_di_node, tag_base_type, @@ -747,9 +747,9 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( ) } -/// This is a helper function shared between enums and generators that makes sure fields have the +/// This is a helper function shared between enums and coroutines that makes sure fields have the /// expect names. -fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( +fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index d3239d5c358..df1df6d197e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -6,11 +6,11 @@ use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::GeneratorLayout, + mir::CoroutineLayout, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - AdtDef, GeneratorArgs, Ty, VariantDef, + AdtDef, CoroutineArgs, Ty, VariantDef, }, }; use rustc_span::Symbol; @@ -66,14 +66,14 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( } } -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { if cpp_like_debuginfo(cx.tcx) { - cpp_like::build_generator_di_node(cx, unique_type_id) + cpp_like::build_coroutine_di_node(cx, unique_type_id) } else { - native::build_generator_di_node(cx, unique_type_id) + native::build_coroutine_di_node(cx, unique_type_id) } } @@ -101,13 +101,13 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( } } -/// Extract the type with which we want to describe the tag of the given enum or generator. +/// Extract the type with which we want to describe the tag of the given enum or coroutine. fn tag_base_type<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, ) -> Ty<'tcx> { debug_assert!(match enum_type_and_layout.ty.kind() { - ty::Generator(..) => true, + ty::Coroutine(..) => true, ty::Adt(adt_def, _) => adt_def.is_enum(), _ => false, }); @@ -300,8 +300,8 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( .di_node } -/// Build the struct type for describing a single generator state. -/// See [build_generator_variant_struct_type_di_node]. +/// Build the struct type for describing a single coroutine state. +/// See [build_coroutine_variant_struct_type_di_node]. /// /// ```txt /// @@ -317,25 +317,25 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( /// ---> DW_TAG_structure_type (type of variant 3) /// /// ``` -pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( +pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, variant_index: VariantIdx, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, - generator_layout: &GeneratorLayout<'tcx>, + coroutine_type_and_layout: TyAndLayout<'tcx>, + coroutine_type_di_node: &'ll DIType, + coroutine_layout: &CoroutineLayout<'tcx>, common_upvar_names: &IndexSlice<FieldIdx, Symbol>, ) -> &'ll DIType { - let variant_name = GeneratorArgs::variant_name(variant_index); + let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( cx.tcx, - generator_type_and_layout.ty, + coroutine_type_and_layout.ty, variant_index, ); - let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); + let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let generator_args = match generator_type_and_layout.ty.kind() { - ty::Generator(_, args, _) => args.as_generator(), + let coroutine_args = match coroutine_type_and_layout.ty.kind() { + ty::Coroutine(_, args, _) => args.as_coroutine(), _ => unreachable!(), }; @@ -346,17 +346,17 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &variant_name, - size_and_align_of(generator_type_and_layout), - Some(generator_type_di_node), + size_and_align_of(coroutine_type_and_layout), + Some(coroutine_type_di_node), DIFlags::FlagZero, ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) .map(|field_index| { - let generator_saved_local = generator_layout.variant_fields[variant_index] + let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; - let field_name_maybe = generator_layout.field_names[generator_saved_local]; + let field_name_maybe = coroutine_layout.field_names[coroutine_saved_local]; let field_name = field_name_maybe .as_ref() .map(|s| Cow::from(s.as_str())) @@ -377,7 +377,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .collect(); // Fields that are common to all states - let common_fields: SmallVec<_> = generator_args + let common_fields: SmallVec<_> = coroutine_args .prefix_tys() .iter() .zip(common_upvar_names) @@ -388,7 +388,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( variant_struct_type_di_node, upvar_name.as_str(), cx.size_and_align_of(upvar_ty), - generator_type_and_layout.fields.offset(index), + coroutine_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, upvar_ty), ) @@ -397,7 +397,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() }, - |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), + |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) .di_node } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index feac40d8c30..7eff52b857f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -110,12 +110,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ) } -/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for +/// Build the debuginfo node for a coroutine environment. It looks the same as the debuginfo for /// an enum. See [build_enum_type_di_node] for more information. /// /// ```txt /// -/// ---> DW_TAG_structure_type (top-level type for the generator) +/// ---> DW_TAG_structure_type (top-level type for the coroutine) /// DW_TAG_variant_part (variant part) /// DW_AT_discr (reference to discriminant DW_TAG_member) /// DW_TAG_member (discriminant member) @@ -127,21 +127,21 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( /// DW_TAG_structure_type (type of variant 3) /// /// ``` -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let &ty::Generator(generator_def_id, _, _) = generator_type.kind() else { - bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) + let coroutine_type = unique_type_id.expect_ty(); + let &ty::Coroutine(coroutine_def_id, _, _) = coroutine_type.kind() else { + bug!("build_coroutine_di_node() called with non-coroutine type: `{:?}`", coroutine_type) }; - let containing_scope = get_namespace_for_item(cx, generator_def_id); - let generator_type_and_layout = cx.layout_of(generator_type); + let containing_scope = get_namespace_for_item(cx, coroutine_def_id); + let coroutine_type_and_layout = cx.layout_of(coroutine_type); - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + debug_assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); type_map::build_type_with_children( cx, @@ -149,37 +149,37 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( cx, Stub::Struct, unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), + &coroutine_type_name, + size_and_align_of(coroutine_type_and_layout), Some(containing_scope), DIFlags::FlagZero, ), - |cx, generator_type_di_node| { - let generator_layout = - cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); + |cx, coroutine_type_di_node| { + let coroutine_layout = + cx.tcx.optimized_mir(coroutine_def_id).coroutine_layout().unwrap(); let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = - generator_type_and_layout.variants + coroutine_type_and_layout.variants else { bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout + "Encountered coroutine with non-direct-tag layout: {:?}", + coroutine_type_and_layout ) }; let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); + cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() .map(|variant_index| { // FIXME: This is problematic because just a number is not a valid identifier. - // GeneratorArgs::variant_name(variant_index), would be consistent + // CoroutineArgs::variant_name(variant_index), would be consistent // with enums? let variant_name = format!("{}", variant_index.as_usize()).into(); - let span = generator_layout.variant_source_info[variant_index].span; + let span = coroutine_layout.variant_source_info[variant_index].span; let source_info = if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); Some((file_metadata(cx, &loc.file), loc.line)) @@ -191,12 +191,12 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( variant_index, variant_name, variant_struct_type_di_node: - super::build_generator_variant_struct_type_di_node( + super::build_coroutine_variant_struct_type_di_node( cx, variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_layout, &common_upvar_names, ), source_info, @@ -206,18 +206,18 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( smallvec![build_enum_variant_part_di_node( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, &variant_struct_type_di_nodes[..], )] }, - // We don't seem to be emitting generic args on the generator type, it seems. Rather + // We don't seem to be emitting generic args on the coroutine type, it seems. Rather // they get attached to the struct type of each variant. NO_GENERICS, ) } -/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: +/// Builds the DW_TAG_variant_part of an enum or coroutine debuginfo node: /// /// ```txt /// DW_TAG_structure_type (top-level type for enum) @@ -306,11 +306,11 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( /// ``` fn build_discr_member_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_or_generator_type_and_layout: TyAndLayout<'tcx>, - enum_or_generator_type_di_node: &'ll DIType, + enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>, + enum_or_coroutine_type_di_node: &'ll DIType, ) -> Option<&'ll DIType> { - let tag_name = match enum_or_generator_type_and_layout.ty.kind() { - ty::Generator(..) => "__state", + let tag_name = match enum_or_coroutine_type_and_layout.ty.kind() { + ty::Coroutine(..) => "__state", _ => "", }; @@ -320,14 +320,14 @@ fn build_discr_member_di_node<'ll, 'tcx>( // In LLVM IR the wrong scope will be listed but when DWARF is // generated from it, the DW_TAG_member will be a child the // DW_TAG_variant_part. - let containing_scope = enum_or_generator_type_di_node; + let containing_scope = enum_or_coroutine_type_di_node; - match enum_or_generator_type_and_layout.layout.variants() { + match enum_or_coroutine_type_and_layout.layout.variants() { // A single-variant enum has no discriminant. &Variants::Single { .. } => None, &Variants::Multiple { tag_field, .. } => { - let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); + let tag_base_type = tag_base_type(cx, enum_or_coroutine_type_and_layout); let (size, align) = cx.size_and_align_of(tag_base_type); unsafe { @@ -340,7 +340,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( UNKNOWN_LINE_NUMBER, size.bits(), align.bits() as u32, - enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), + enum_or_coroutine_type_and_layout.fields.offset(tag_field).bits(), DIFlags::FlagArtificial, type_di_node(cx, tag_base_type), )) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index e30622cbdce..1aec65cf949 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -43,7 +43,7 @@ pub(super) enum UniqueTypeId<'tcx> { /// The ID of a regular type as it shows up at the language level. Ty(Ty<'tcx>, private::HiddenZst), /// The ID for the single DW_TAG_variant_part nested inside the top-level - /// DW_TAG_structure_type that describes enums and generators. + /// DW_TAG_structure_type that describes enums and coroutines. VariantPart(Ty<'tcx>, private::HiddenZst), /// The ID for the artificial struct type describing a single enum variant. VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index d874b3ab99d..4832b147a54 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -50,7 +50,6 @@ mod utils; pub use self::create_scope_map::compute_mir_scopes; pub use self::metadata::build_global_var_di_node; -pub use self::metadata::extend_scope_to_file; #[allow(non_upper_case_globals)] const DW_TAG_auto_variable: c_uint = 0x100; @@ -342,7 +341,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // We look up the generics of the enclosing function and truncate the args // to their length in order to cut off extra stuff that might be in there for - // closures or generators. + // closures or coroutines. let generics = tcx.generics_of(enclosing_fn_def_id); let args = instance.args.truncate_to(tcx, generics); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 7a390d35a2b..8a6a5f79b3b 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -8,12 +8,13 @@ #![cfg_attr(not(bootstrap), feature(rustdoc_internals))] #![cfg_attr(not(bootstrap), doc(rust_logo))] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(exact_size_is_empty)] #![feature(extern_types)] #![feature(hash_raw_entry)] #![feature(iter_intersperse)] #![feature(let_chains)] +#![feature(min_specialization)] #![feature(never_type)] -#![feature(slice_group_by)] #![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 38e8220569a..01e82339664 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -6,7 +6,6 @@ use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index fd4c9572af2..712b6ed5333 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -42,7 +42,7 @@ fn uncached_llvm_type<'a, 'tcx>( // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { @@ -54,10 +54,10 @@ fn uncached_llvm_type<'a, 'tcx>( write!(&mut name, "::{}", def.variant(index).name).unwrap(); } } - if let (&ty::Generator(_, _, _), &Variants::Single { index }) = + if let (&ty::Coroutine(_, _, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { - write!(&mut name, "::{}", ty::GeneratorArgs::variant_name(index)).unwrap(); + write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap(); } Some(name) } |