diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs | 93 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 45 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/type_of.rs | 7 |
6 files changed, 95 insertions, 78 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ac6d8f84142..4f9b86ec20a 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,6 +3,7 @@ use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -1225,9 +1226,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn do_not_inline(&mut self, llret: &'ll Value) { - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + if llvm_util::get_version() < (17, 0, 2) { + // Work around https://github.com/llvm/llvm-project/issues/66984. + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + } else { + // Cleanup is always the cold path. + let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index e83110dcad4..55f43aa5341 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -11,7 +11,7 @@ pub struct Expression { lhs: Operand, op: Op, rhs: Operand, - region: Option<CodeRegion>, + code_regions: Vec<CodeRegion>, } /// Collects all of the coverage regions associated with (a) injected counters, (b) counter @@ -30,7 +30,7 @@ pub struct FunctionCoverage<'tcx> { instance: Instance<'tcx>, source_hash: u64, is_used: bool, - counters: IndexVec<CounterId, Option<CodeRegion>>, + counters: IndexVec<CounterId, Option<Vec<CodeRegion>>>, expressions: IndexVec<ExpressionId, Option<Expression>>, unreachable_regions: Vec<CodeRegion>, } @@ -77,28 +77,40 @@ impl<'tcx> FunctionCoverage<'tcx> { } } - /// Adds a code region to be counted by an injected counter intrinsic. - pub fn add_counter(&mut self, id: CounterId, region: CodeRegion) { - if let Some(previous_region) = self.counters[id].replace(region.clone()) { - assert_eq!(previous_region, region, "add_counter: code region for id changed"); + /// Adds code regions to be counted by an injected counter intrinsic. + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_counter(&mut self, id: CounterId, code_regions: &[CodeRegion]) { + if code_regions.is_empty() { + return; + } + + let slot = &mut self.counters[id]; + match slot { + None => *slot = Some(code_regions.to_owned()), + // If this counter ID slot has already been filled, it should + // contain identical information. + Some(ref previous_regions) => assert_eq!( + previous_regions, code_regions, + "add_counter: code regions for id changed" + ), } } + /// Adds information about a coverage expression, along with zero or more + /// code regions mapped to that expression. + /// /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other /// expressions. These are tracked as separate variants of `Operand`, so there is no ambiguity /// between operands that are counter IDs and operands that are expression IDs. - pub fn add_counter_expression( + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_counter_expression( &mut self, expression_id: ExpressionId, lhs: Operand, op: Op, rhs: Operand, - region: Option<CodeRegion>, + code_regions: &[CodeRegion], ) { - debug!( - "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", - expression_id, lhs, op, rhs, region - ); debug_assert!( expression_id.as_usize() < self.expressions.len(), "expression_id {} is out of range for expressions.len() = {} @@ -107,23 +119,25 @@ impl<'tcx> FunctionCoverage<'tcx> { self.expressions.len(), self, ); - if let Some(previous_expression) = self.expressions[expression_id].replace(Expression { - lhs, - op, - rhs, - region: region.clone(), - }) { - assert_eq!( - previous_expression, - Expression { lhs, op, rhs, region }, + + let expression = Expression { lhs, op, rhs, code_regions: code_regions.to_owned() }; + let slot = &mut self.expressions[expression_id]; + match slot { + None => *slot = Some(expression), + // If this expression ID slot has already been filled, it should + // contain identical information. + Some(ref previous_expression) => assert_eq!( + previous_expression, &expression, "add_counter_expression: expression for id changed" - ); + ), } } - /// Add a region that will be marked as "unreachable", with a constant "zero counter". - pub fn add_unreachable_region(&mut self, region: CodeRegion) { - self.unreachable_regions.push(region) + /// Adds regions that will be marked as "unreachable", with a constant "zero counter". + #[instrument(level = "debug", skip(self))] + pub(crate) fn add_unreachable_regions(&mut self, code_regions: &[CodeRegion]) { + assert!(!code_regions.is_empty(), "unreachable regions always have code regions"); + self.unreachable_regions.extend_from_slice(code_regions); } /// Perform some simplifications to make the final coverage mappings @@ -212,11 +226,16 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.counters.iter_enumerated().filter_map(|(index, entry)| { - // Option::map() will return None to filter out missing counters. This may happen - // if, for example, a MIR-instrumented counter is removed during an optimization. - entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) - }) + self.counters + .iter_enumerated() + // Filter out counter IDs that we never saw during MIR traversal. + // This can happen if a counter was optimized out by MIR transforms + // (and replaced with `CoverageKind::Unreachable` instead). + .filter_map(|(id, maybe_code_regions)| Some((id, maybe_code_regions.as_ref()?))) + .flat_map(|(id, code_regions)| { + let counter = Counter::counter_value_reference(id); + code_regions.iter().map(move |region| (counter, region)) + }) } /// Convert this function's coverage expression data into a form that can be @@ -254,13 +273,17 @@ impl<'tcx> FunctionCoverage<'tcx> { fn expression_regions(&self) -> Vec<(Counter, &CodeRegion)> { // Find all of the expression IDs that weren't optimized out AND have - // an attached code region, and return the corresponding mapping as a - // counter/region pair. + // one or more attached code regions, and return the corresponding + // mappings as counter/region pairs. self.expressions .iter_enumerated() - .filter_map(|(id, expression)| { - let code_region = expression.as_ref()?.region.as_ref()?; - Some((Counter::expression(id), code_region)) + .filter_map(|(id, maybe_expression)| { + let code_regions = &maybe_expression.as_ref()?.code_regions; + Some((id, code_regions)) + }) + .flat_map(|(id, code_regions)| { + let counter = Counter::expression(id); + code_regions.iter().map(move |code_region| (counter, code_region)) }) .collect::<Vec<_>>() } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index c70cb670e96..dd2ce9b525b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -89,9 +89,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// `function_coverage_map` (keyed by function `Instance`) during codegen. /// But in this case, since the unused function was _not_ previously /// codegenned, collect the coverage `CodeRegion`s from the MIR and add - /// them. The first `CodeRegion` is used to add a single counter, with the - /// same counter ID used in the injected `instrprof.increment` intrinsic - /// call. Since the function is never called, all other `CodeRegion`s can be + /// them. Since the function is never called, all of its `CodeRegion`s can be /// added as `unreachable_region`s. fn define_unused_fn(&self, def_id: DefId) { let instance = declare_unused_fn(self, def_id); @@ -110,25 +108,15 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { .entry(instance) .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); - let Coverage { kind, code_region } = coverage.clone(); - match kind { + let Coverage { kind, code_regions } = coverage; + match *kind { CoverageKind::Counter { function_source_hash, id } => { debug!( "ensuring function source hash is set for instance={:?}; function_source_hash={}", instance, function_source_hash, ); func_coverage.set_function_source_hash(function_source_hash); - - if let Some(code_region) = code_region { - // Note: Some counters do not have code regions, but may still be referenced - // from expressions. In that case, don't add the counter to the coverage map, - // but do inject the counter intrinsic. - debug!( - "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", - instance, id, code_region, - ); - func_coverage.add_counter(id, code_region); - } + func_coverage.add_counter(id, code_regions); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, // as that needs an exclusive borrow. drop(coverage_map); @@ -146,20 +134,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { bx.instrprof_increment(fn_name, hash, num_counters, index); } CoverageKind::Expression { id, lhs, op, rhs } => { - debug!( - "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; region: {:?}", - instance, id, lhs, op, rhs, code_region, - ); - func_coverage.add_counter_expression(id, lhs, op, rhs, code_region); + func_coverage.add_counter_expression(id, lhs, op, rhs, code_regions); } CoverageKind::Unreachable => { - let code_region = - code_region.expect("unreachable regions always have code regions"); - debug!( - "adding unreachable code to coverage_map: instance={:?}, at {:?}", - instance, code_region, - ); - func_coverage.add_unreachable_region(code_region); + func_coverage.add_unreachable_regions(code_regions); } } } @@ -227,14 +205,9 @@ fn add_unused_function_coverage<'tcx>( let tcx = cx.tcx; let mut function_coverage = FunctionCoverage::unused(tcx, instance); - for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { - if index == 0 { - // Insert at least one real counter so the LLVM CoverageMappingReader will find expected - // definitions. - function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); - } else { - function_coverage.add_unreachable_region(code_region.clone()); - } + for &code_region in tcx.covered_code_regions(def_id) { + let code_region = std::slice::from_ref(code_region); + function_coverage.add_unreachable_regions(code_region); } if let Some(coverage_context) = cx.coverage_context() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ed938761694..11874898a5a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -335,12 +335,20 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let (size, align) = match fn_ty.kind() { + ty::FnDef(..) => (0, 1), + ty::FnPtr(..) => ( + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + ), + _ => unreachable!(), + }; let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), fn_di_node, - cx.tcx.data_layout.pointer_size.bits(), - cx.tcx.data_layout.pointer_align.abi.bits() as u32, + size, + align, 0, // Ignore DWARF address space. name.as_ptr().cast(), name.len(), diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 8db6195d931..06b7703672f 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -112,7 +112,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - /// Return a LLVM type that has at most the required alignment, + /// Return an LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index dcc62d314ff..fd4c9572af2 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -397,7 +397,12 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // extracts all the individual values. let ety = element.llvm_type(cx); - return Some(cx.type_vector(ety, *count)); + if *count == 1 { + // Emitting `<1 x T>` would be silly; just use the scalar. + return Some(ety); + } else { + return Some(cx.type_vector(ety, *count)); + } } // FIXME: The above only handled integer arrays; surely more things |