diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 29 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs | 28 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs | 182 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 226 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs | 200 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 3 |
13 files changed, 343 insertions, 399 deletions
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index d1804cb49ad..3722d4350a2 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -442,6 +442,14 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { ); } } + + fn mangled_name(&self, instance: Instance<'tcx>) -> String { + let llval = self.get_fn(instance); + llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8") + } } pub(crate) fn inline_asm_call<'ll>( @@ -504,14 +512,13 @@ pub(crate) fn inline_asm_call<'ll>( let key = "srcloc"; let kind = llvm::LLVMGetMDKindIDInContext( bx.llcx, - key.as_ptr() as *const c_char, + key.as_ptr().cast::<c_char>(), key.len() as c_uint, ); - // srcloc contains one integer for each line of assembly code. - // Unfortunately this isn't enough to encode a full span so instead - // we just encode the start position of each line. - // FIXME: Figure out a way to pass the entire line spans. + // `srcloc` contains one 64-bit integer for each line of assembly code, + // where the lower 32 bits hold the lo byte position and the upper 32 bits + // hold the hi byte position. let mut srcloc = vec![]; if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { // LLVM inserts an extra line to add the ".intel_syntax", so add @@ -521,13 +528,13 @@ pub(crate) fn inline_asm_call<'ll>( // due to the asm template string coming from a macro. LLVM will // default to the first srcloc for lines that don't have an // associated srcloc. - srcloc.push(llvm::LLVMValueAsMetadata(bx.const_i32(0))); + srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0))); } - srcloc.extend( - line_spans - .iter() - .map(|span| llvm::LLVMValueAsMetadata(bx.const_i32(span.lo().to_u32() as i32))), - ); + srcloc.extend(line_spans.iter().map(|span| { + llvm::LLVMValueAsMetadata(bx.const_u64( + u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32), + )) + })); let md = llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()); let md = llvm::LLVMMetadataAsValue(&bx.llcx, md); llvm::LLVMSetMetadata(call, kind, md); diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index cb958c1d4d7..5552a241060 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -395,17 +395,9 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( to_add.push(MemoryEffects::None.create_attr(cx.llcx)); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { - to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); - // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. - // And it is a module-level attribute, so the alternative is pulling naked functions into - // new LLVM modules. Otherwise LLVM's "naked" functions come with endbr prefixes per - // https://github.com/rust-lang/rust/issues/98768 - to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); - if llvm_util::get_version() < (19, 0, 0) { - // Prior to LLVM 19, branch-target-enforcement was disabled by setting the attribute to - // the string "false". Now it is disabled by absence of the attribute. - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); - } + // do nothing; a naked function is converted into an extern function + // and a global assembly block. LLVM's support for naked functions is + // not used. } else { // Do not set sanitizer attributes for naked functions. to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index bde6668929c..66ca4e2b473 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -25,8 +25,8 @@ use rustc_session::Session; use rustc_session::config::{ self, Lto, OutputType, Passes, RemapPathScopeComponents, SplitDwarfKind, SwitchWithOptPath, }; -use rustc_span::InnerSpan; use rustc_span::symbol::sym; +use rustc_span::{BytePos, InnerSpan, Pos, SpanData, SyntaxContext}; use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel}; use tracing::debug; @@ -415,21 +415,32 @@ fn report_inline_asm( cgcx: &CodegenContext<LlvmCodegenBackend>, msg: String, level: llvm::DiagnosticLevel, - mut cookie: u64, + cookie: u64, source: Option<(String, Vec<InnerSpan>)>, ) { // In LTO build we may get srcloc values from other crates which are invalid // since they use a different source map. To be safe we just suppress these // in LTO builds. - if matches!(cgcx.lto, Lto::Fat | Lto::Thin) { - cookie = 0; - } + let span = if cookie == 0 || matches!(cgcx.lto, Lto::Fat | Lto::Thin) { + SpanData::default() + } else { + let lo = BytePos::from_u32(cookie as u32); + let hi = BytePos::from_u32((cookie >> 32) as u32); + SpanData { + lo, + // LLVM version < 19 silently truncates the cookie to 32 bits in some situations. + hi: if hi.to_u32() != 0 { hi } else { lo }, + ctxt: SyntaxContext::root(), + parent: None, + } + }; let level = match level { llvm::DiagnosticLevel::Error => Level::Error, llvm::DiagnosticLevel::Warning => Level::Warning, llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, }; - cgcx.diag_emitter.inline_asm_error(cookie.try_into().unwrap(), msg, level, source); + let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string(); + cgcx.diag_emitter.inline_asm_error(span, msg, level, source); } unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8218126ea29..c602d99ff9d 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -159,6 +159,16 @@ pub(crate) unsafe fn create_module<'ll>( // See https://github.com/llvm/llvm-project/pull/112084 target_data_layout = target_data_layout.replace("-i128:128", ""); } + if sess.target.arch.starts_with("powerpc64") { + // LLVM 20 updates the powerpc64 layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/118004 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } + if sess.target.arch.starts_with("wasm32") || sess.target.arch.starts_with("wasm64") { + // LLVM 20 updates the wasm(32|64) layout to correctly align 128 bit integers to 128 bit. + // See https://github.com/llvm/llvm-project/pull/119204 + target_data_layout = target_data_layout.replace("-i128:128", ""); + } } // Ensure the data-layout values hardcoded remain the defaults. diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index a6e07ea2a60..1f133141c18 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -152,6 +152,34 @@ impl CoverageSpan { } } +/// Holds tables of the various region types in one struct. +/// +/// Don't pass this struct across FFI; pass the individual region tables as +/// pointer/length pairs instead. +/// +/// Each field name has a `_regions` suffix for improved readability after +/// exhaustive destructing, which ensures that all region types are handled. +#[derive(Clone, Debug, Default)] +pub(crate) struct Regions { + pub(crate) code_regions: Vec<CodeRegion>, + pub(crate) branch_regions: Vec<BranchRegion>, + pub(crate) mcdc_branch_regions: Vec<MCDCBranchRegion>, + pub(crate) mcdc_decision_regions: Vec<MCDCDecisionRegion>, +} + +impl Regions { + /// Returns true if none of this structure's tables contain any regions. + pub(crate) fn has_no_regions(&self) -> bool { + let Self { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = + self; + + code_regions.is_empty() + && branch_regions.is_empty() + && mcdc_branch_regions.is_empty() + && mcdc_decision_regions.is_empty() + } +} + /// Must match the layout of `LLVMRustCoverageCodeRegion`. #[derive(Clone, Debug)] #[repr(C)] diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index 99c2d12b261..086cf1f44a0 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -62,11 +62,10 @@ pub(crate) fn write_filenames_to_buffer<'a>( pub(crate) fn write_function_mappings_to_buffer( virtual_file_mapping: &[u32], expressions: &[ffi::CounterExpression], - code_regions: &[ffi::CodeRegion], - branch_regions: &[ffi::BranchRegion], - mcdc_branch_regions: &[ffi::MCDCBranchRegion], - mcdc_decision_regions: &[ffi::MCDCDecisionRegion], + regions: &ffi::Regions, ) -> Vec<u8> { + let ffi::Regions { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = + regions; llvm::build_byte_buffer(|buffer| unsafe { llvm::LLVMRustCoverageWriteFunctionMappingsToBuffer( virtual_file_mapping.as_ptr(), diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 0752c718c70..c5d1ebdfe7c 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -1,159 +1,38 @@ use rustc_data_structures::captures::Captures; -use rustc_data_structures::fx::FxIndexSet; -use rustc_index::bit_set::BitSet; -use rustc_middle::mir::CoverageIdsInfo; use rustc_middle::mir::coverage::{ - CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, MappingKind, Op, + CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op, SourceRegion, }; -use rustc_middle::ty::Instance; -use tracing::debug; use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; -/// Holds all of the coverage mapping data associated with a function instance, -/// collected during traversal of `Coverage` statements in the function's MIR. -#[derive(Debug)] -pub(crate) struct FunctionCoverageCollector<'tcx> { - /// Coverage info that was attached to this function by the instrumentor. - function_coverage_info: &'tcx FunctionCoverageInfo, - ids_info: &'tcx CoverageIdsInfo, - is_used: bool, +pub(crate) struct FunctionCoverage<'tcx> { + pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, + /// If `None`, the corresponding function is unused. + ids_info: Option<&'tcx CoverageIdsInfo>, } -impl<'tcx> FunctionCoverageCollector<'tcx> { - /// Creates a new set of coverage data for a used (called) function. - pub(crate) fn new( - instance: Instance<'tcx>, - function_coverage_info: &'tcx FunctionCoverageInfo, - ids_info: &'tcx CoverageIdsInfo, - ) -> Self { - Self::create(instance, function_coverage_info, ids_info, true) - } - - /// Creates a new set of coverage data for an unused (never called) function. - pub(crate) fn unused( - instance: Instance<'tcx>, - function_coverage_info: &'tcx FunctionCoverageInfo, - ids_info: &'tcx CoverageIdsInfo, - ) -> Self { - Self::create(instance, function_coverage_info, ids_info, false) - } - - fn create( - instance: Instance<'tcx>, +impl<'tcx> FunctionCoverage<'tcx> { + pub(crate) fn new_used( function_coverage_info: &'tcx FunctionCoverageInfo, ids_info: &'tcx CoverageIdsInfo, - is_used: bool, ) -> Self { - let num_counters = function_coverage_info.num_counters; - let num_expressions = function_coverage_info.expressions.len(); - debug!( - "FunctionCoverage::create(instance={instance:?}) has \ - num_counters={num_counters}, num_expressions={num_expressions}, is_used={is_used}" - ); - - Self { function_coverage_info, ids_info, is_used } - } - - /// Identify expressions that will always have a value of zero, and note - /// their IDs in [`ZeroExpressions`]. Mappings that refer to a zero expression - /// can instead become mappings to a constant zero value. - /// - /// This method mainly exists to preserve the simplifications that were - /// already being performed by the Rust-side expression renumbering, so that - /// the resulting coverage mappings don't get worse. - fn identify_zero_expressions(&self) -> ZeroExpressions { - // The set of expressions that either were optimized out entirely, or - // have zero as both of their operands, and will therefore always have - // a value of zero. Other expressions that refer to these as operands - // can have those operands replaced with `CovTerm::Zero`. - let mut zero_expressions = ZeroExpressions::default(); - - // Simplify a copy of each expression based on lower-numbered expressions, - // and then update the set of always-zero expressions if necessary. - // (By construction, expressions can only refer to other expressions - // that have lower IDs, so one pass is sufficient.) - for (id, expression) in self.function_coverage_info.expressions.iter_enumerated() { - if !self.is_used || !self.ids_info.expressions_seen.contains(id) { - // If an expression was not seen, it must have been optimized away, - // so any operand that refers to it can be replaced with zero. - zero_expressions.insert(id); - continue; - } - - // We don't need to simplify the actual expression data in the - // expressions list; we can just simplify a temporary copy and then - // use that to update the set of always-zero expressions. - let Expression { mut lhs, op, mut rhs } = *expression; - - // If an expression has an operand that is also an expression, the - // operand's ID must be strictly lower. This is what lets us find - // all zero expressions in one pass. - let assert_operand_expression_is_lower = |operand_id: ExpressionId| { - assert!( - operand_id < id, - "Operand {operand_id:?} should be less than {id:?} in {expression:?}", - ) - }; - - // If an operand refers to a counter or expression that is always - // zero, then that operand can be replaced with `CovTerm::Zero`. - let maybe_set_operand_to_zero = |operand: &mut CovTerm| { - if let CovTerm::Expression(id) = *operand { - assert_operand_expression_is_lower(id); - } - - if is_zero_term(&self.ids_info.counters_seen, &zero_expressions, *operand) { - *operand = CovTerm::Zero; - } - }; - maybe_set_operand_to_zero(&mut lhs); - maybe_set_operand_to_zero(&mut rhs); - - // Coverage counter values cannot be negative, so if an expression - // involves subtraction from zero, assume that its RHS must also be zero. - // (Do this after simplifications that could set the LHS to zero.) - if lhs == CovTerm::Zero && op == Op::Subtract { - rhs = CovTerm::Zero; - } - - // After the above simplifications, if both operands are zero, then - // we know that this expression is always zero too. - if lhs == CovTerm::Zero && rhs == CovTerm::Zero { - zero_expressions.insert(id); - } - } - - zero_expressions + Self { function_coverage_info, ids_info: Some(ids_info) } } - pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> { - let zero_expressions = self.identify_zero_expressions(); - let FunctionCoverageCollector { function_coverage_info, ids_info, is_used, .. } = self; - - FunctionCoverage { function_coverage_info, ids_info, is_used, zero_expressions } + pub(crate) fn new_unused(function_coverage_info: &'tcx FunctionCoverageInfo) -> Self { + Self { function_coverage_info, ids_info: None } } -} - -pub(crate) struct FunctionCoverage<'tcx> { - pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, - ids_info: &'tcx CoverageIdsInfo, - is_used: bool, - - zero_expressions: ZeroExpressions, -} -impl<'tcx> FunctionCoverage<'tcx> { /// Returns true for a used (called) function, and false for an unused function. pub(crate) fn is_used(&self) -> bool { - self.is_used + self.ids_info.is_some() } /// Return the source hash, generated from the HIR node structure, and used to indicate whether /// or not the source code structure changed between different compilations. pub(crate) fn source_hash(&self) -> u64 { - if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } + if self.is_used() { self.function_coverage_info.function_source_hash } else { 0 } } /// Convert this function's coverage expression data into a form that can be @@ -196,37 +75,10 @@ impl<'tcx> FunctionCoverage<'tcx> { } fn is_zero_term(&self, term: CovTerm) -> bool { - !self.is_used || is_zero_term(&self.ids_info.counters_seen, &self.zero_expressions, term) - } -} - -/// Set of expression IDs that are known to always evaluate to zero. -/// Any mapping or expression operand that refers to these expressions can have -/// that reference replaced with a constant zero value. -#[derive(Default)] -struct ZeroExpressions(FxIndexSet<ExpressionId>); - -impl ZeroExpressions { - fn insert(&mut self, id: ExpressionId) { - self.0.insert(id); - } - - fn contains(&self, id: ExpressionId) -> bool { - self.0.contains(&id) - } -} - -/// Returns `true` if the given term is known to have a value of zero, taking -/// into account knowledge of which counters are unused and which expressions -/// are always zero. -fn is_zero_term( - counters_seen: &BitSet<CounterId>, - zero_expressions: &ZeroExpressions, - term: CovTerm, -) -> bool { - match term { - CovTerm::Zero => true, - CovTerm::Counter(id) => !counters_seen.contains(id), - CovTerm::Expression(id) => zero_expressions.contains(id), + match self.ids_info { + Some(ids_info) => ids_info.is_zero_term(term), + // This function is unused, so all coverage counters/expressions are zero. + None => true, + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 8c24579fa7c..a573a37beb3 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,4 +1,3 @@ -use std::ffi::CString; use std::iter; use itertools::Itertools as _; @@ -9,21 +8,22 @@ use rustc_codegen_ssa::traits::{ use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; -use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_session::RemapFileNameExt; use rustc_session::config::RemapPathScopeComponents; use rustc_span::def_id::DefIdSet; use rustc_span::{Span, Symbol}; -use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; -use crate::coverageinfo::{ffi, llvm_cov}; +use crate::coverageinfo::llvm_cov; +use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::mapgen::covfun::prepare_covfun_record; use crate::llvm; +mod covfun; + /// Generates and exports the coverage map, which is embedded in special /// linker sections in the final binary. /// @@ -63,16 +63,11 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { None => return, }; if function_coverage_map.is_empty() { - // This module has no functions with coverage instrumentation + // This CGU has no functions with coverage instrumentation. return; } - let function_coverage_entries = function_coverage_map - .into_iter() - .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) - .collect::<Vec<_>>(); - - let all_file_names = function_coverage_entries + let all_file_names = function_coverage_map .iter() .map(|(_, fn_cov)| fn_cov.function_coverage_info.body_span) .map(|span| span_file_name(tcx, span)); @@ -85,47 +80,28 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let filenames_val = cx.const_bytes(&filenames_buffer); let filenames_ref = llvm_cov::hash_bytes(&filenames_buffer); - // Generate the coverage map header, which contains the filenames used by - // this CGU's coverage mappings, and store it in a well-known global. - generate_covmap_record(cx, covmap_version, filenames_size, filenames_val); - let mut unused_function_names = Vec::new(); - // Encode coverage mappings and generate function records - for (instance, function_coverage) in function_coverage_entries { - debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); - - let mangled_function_name = tcx.symbol_name(instance).name; - let source_hash = function_coverage.source_hash(); - let is_used = function_coverage.is_used(); - - let coverage_mapping_buffer = - encode_mappings_for_function(tcx, &global_file_table, &function_coverage); + let covfun_records = function_coverage_map + .into_iter() + .filter_map(|(instance, function_coverage)| { + prepare_covfun_record(tcx, &global_file_table, instance, &function_coverage) + }) + .collect::<Vec<_>>(); - if coverage_mapping_buffer.is_empty() { - if function_coverage.is_used() { - bug!( - "A used function should have had coverage mapping data but did not: {}", - mangled_function_name - ); - } else { - debug!("unused function had no coverage mapping data: {}", mangled_function_name); - continue; - } - } + // If there are no covfun records for this CGU, don't generate a covmap record. + // Emitting a covmap record without any covfun records causes `llvm-cov` to + // fail when generating coverage reports, and if there are no covfun records + // then the covmap record isn't useful anyway. + // This should prevent a repeat of <https://github.com/rust-lang/rust/issues/133606>. + if covfun_records.is_empty() { + return; + } - if !is_used { - unused_function_names.push(mangled_function_name); - } + for covfun in &covfun_records { + unused_function_names.extend(covfun.mangled_function_name_if_unused()); - generate_covfun_record( - cx, - mangled_function_name, - source_hash, - filenames_ref, - coverage_mapping_buffer, - is_used, - ); + covfun::generate_covfun_record(cx, filenames_ref, covfun) } // For unused functions, we need to take their mangled names and store them @@ -146,6 +122,11 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { llvm::set_linkage(array, llvm::Linkage::InternalLinkage); llvm::set_initializer(array, initializer); } + + // Generate the coverage map header, which contains the filenames used by + // this CGU's coverage mappings, and store it in a well-known global. + // (This is skipped if we returned early due to having no covfun records.) + generate_covmap_record(cx, covmap_version, filenames_size, filenames_val); } /// Maps "global" (per-CGU) file ID numbers to their underlying filenames. @@ -213,7 +194,7 @@ rustc_index::newtype_index! { /// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) /// file IDs. -#[derive(Default)] +#[derive(Debug, Default)] struct VirtualFileMapping { local_to_global: IndexVec<LocalFileId, GlobalFileId>, global_to_local: FxIndexMap<GlobalFileId, LocalFileId>, @@ -227,10 +208,10 @@ impl VirtualFileMapping { .or_insert_with(|| self.local_to_global.push(global_file_id)) } - fn into_vec(self) -> Vec<u32> { - // This conversion should be optimized away to ~zero overhead. - // In any case, it's probably not hot enough to worry about. - self.local_to_global.into_iter().map(|global| global.as_u32()).collect() + fn to_vec(&self) -> Vec<u32> { + // This clone could be avoided by transmuting `&[GlobalFileId]` to `&[u32]`, + // but it isn't hot or expensive enough to justify the extra unsafety. + self.local_to_global.iter().map(|&global| GlobalFileId::as_u32(global)).collect() } } @@ -241,83 +222,6 @@ fn span_file_name(tcx: TyCtxt<'_>, span: Span) -> Symbol { Symbol::intern(&name) } -/// Using the expressions and counter regions collected for a single function, -/// generate the variable-sized payload of its corresponding `__llvm_covfun` -/// entry. The payload is returned as a vector of bytes. -/// -/// Newly-encountered filenames will be added to the global file table. -fn encode_mappings_for_function( - tcx: TyCtxt<'_>, - global_file_table: &GlobalFileTable, - function_coverage: &FunctionCoverage<'_>, -) -> Vec<u8> { - let counter_regions = function_coverage.counter_regions(); - if counter_regions.is_empty() { - return Vec::new(); - } - - let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); - - let mut virtual_file_mapping = VirtualFileMapping::default(); - let mut code_regions = vec![]; - let mut branch_regions = vec![]; - let mut mcdc_branch_regions = vec![]; - let mut mcdc_decision_regions = vec![]; - - // Currently a function's mappings must all be in the same file as its body span. - let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span); - - // Look up the global file ID for that filename. - let global_file_id = global_file_table.global_file_id_for_file_name(file_name); - - // Associate that global file ID with a local file ID for this function. - let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); - debug!(" file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'"); - - // For each counter/region pair in this function+file, convert it to a - // form suitable for FFI. - for (mapping_kind, region) in counter_regions { - debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - let span = ffi::CoverageSpan::from_source_region(local_file_id, region); - match mapping_kind { - MappingKind::Code(term) => { - code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); - } - MappingKind::Branch { true_term, false_term } => { - branch_regions.push(ffi::BranchRegion { - span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), - }); - } - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - mcdc_branch_regions.push(ffi::MCDCBranchRegion { - span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), - mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), - }); - } - MappingKind::MCDCDecision(mcdc_decision_params) => { - mcdc_decision_regions.push(ffi::MCDCDecisionRegion { - span, - mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), - }); - } - } - } - - // Encode the function's coverage mappings into a buffer. - llvm_cov::write_function_mappings_to_buffer( - &virtual_file_mapping.into_vec(), - &expressions, - &code_regions, - &branch_regions, - &mcdc_branch_regions, - &mcdc_decision_regions, - ) -} - /// Generates the contents of the covmap record for this CGU, which mostly /// consists of a header and a list of filenames. The record is then stored /// as a global variable in the `__llvm_covmap` section. @@ -355,61 +259,6 @@ fn generate_covmap_record<'ll>( cx.add_used_global(llglobal); } -/// Generates the contents of the covfun record for this function, which -/// contains the function's coverage mapping data. The record is then stored -/// as a global variable in the `__llvm_covfun` section. -fn generate_covfun_record( - cx: &CodegenCx<'_, '_>, - mangled_function_name: &str, - source_hash: u64, - filenames_ref: u64, - coverage_mapping_buffer: Vec<u8>, - is_used: bool, -) { - // Concatenate the encoded coverage mappings - let coverage_mapping_size = coverage_mapping_buffer.len(); - let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - - let func_name_hash = llvm_cov::hash_bytes(mangled_function_name.as_bytes()); - let func_name_hash_val = cx.const_u64(func_name_hash); - let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); - let source_hash_val = cx.const_u64(source_hash); - let filenames_ref_val = cx.const_u64(filenames_ref); - let func_record_val = cx.const_struct( - &[ - func_name_hash_val, - coverage_mapping_size_val, - source_hash_val, - filenames_ref_val, - coverage_mapping_val, - ], - /*packed=*/ true, - ); - - // Choose a variable name to hold this function's covfun data. - // Functions that are used have a suffix ("u") to distinguish them from - // unused copies of the same function (from different CGUs), so that if a - // linker sees both it won't discard the used copy's data. - let func_record_var_name = - CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) - .unwrap(); - debug!("function record var name: {:?}", func_record_var_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); - llvm::set_initializer(llglobal, func_record_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, cx.covfun_section_name()); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - // <https://llvm.org/docs/CoverageMappingFormat.html> - llvm::set_alignment(llglobal, Align::EIGHT); - if cx.target_spec().supports_comdat() { - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); - } - cx.add_used_global(llglobal); -} - /// Each CGU will normally only emit coverage metadata for the functions that it actually generates. /// But since we don't want unused functions to disappear from coverage reports, we also scan for /// functions that were instrumented but are not participating in codegen. @@ -536,11 +385,6 @@ fn add_unused_function_coverage<'tcx>( ); // An unused function's mappings will all be rewritten to map to zero. - let function_coverage = FunctionCoverageCollector::unused( - instance, - function_coverage_info, - tcx.coverage_ids_info(instance.def), - ); - + let function_coverage = FunctionCoverage::new_unused(function_coverage_info); cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs new file mode 100644 index 00000000000..530e6827f55 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -0,0 +1,200 @@ +//! For each function that was instrumented for coverage, we need to embed its +//! corresponding coverage mapping metadata inside the `__llvm_covfun`[^win] +//! linker section of the final binary. +//! +//! [^win]: On Windows the section name is `.lcovfun`. + +use std::ffi::CString; + +use rustc_abi::Align; +use rustc_codegen_ssa::traits::{ + BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, +}; +use rustc_middle::bug; +use rustc_middle::mir::coverage::MappingKind; +use rustc_middle::ty::{Instance, TyCtxt}; +use rustc_target::spec::HasTargetSpec; +use tracing::debug; + +use crate::common::CodegenCx; +use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::mapgen::{GlobalFileTable, VirtualFileMapping, span_file_name}; +use crate::coverageinfo::{ffi, llvm_cov}; +use crate::llvm; + +/// Intermediate coverage metadata for a single function, used to help build +/// the final record that will be embedded in the `__llvm_covfun` section. +#[derive(Debug)] +pub(crate) struct CovfunRecord<'tcx> { + mangled_function_name: &'tcx str, + source_hash: u64, + is_used: bool, + + virtual_file_mapping: VirtualFileMapping, + expressions: Vec<ffi::CounterExpression>, + regions: ffi::Regions, +} + +impl<'tcx> CovfunRecord<'tcx> { + /// FIXME(Zalathar): Make this the responsibility of the code that determines + /// which functions are unused. + pub(crate) fn mangled_function_name_if_unused(&self) -> Option<&'tcx str> { + (!self.is_used).then_some(self.mangled_function_name) + } +} + +pub(crate) fn prepare_covfun_record<'tcx>( + tcx: TyCtxt<'tcx>, + global_file_table: &GlobalFileTable, + instance: Instance<'tcx>, + function_coverage: &FunctionCoverage<'tcx>, +) -> Option<CovfunRecord<'tcx>> { + let mut covfun = CovfunRecord { + mangled_function_name: tcx.symbol_name(instance).name, + source_hash: function_coverage.source_hash(), + is_used: function_coverage.is_used(), + virtual_file_mapping: VirtualFileMapping::default(), + expressions: function_coverage.counter_expressions().collect::<Vec<_>>(), + regions: ffi::Regions::default(), + }; + + fill_region_tables(tcx, global_file_table, function_coverage, &mut covfun); + + if covfun.regions.has_no_regions() { + if covfun.is_used { + bug!("a used function should have had coverage mapping data but did not: {covfun:?}"); + } else { + debug!(?covfun, "unused function had no coverage mapping data"); + return None; + } + } + + Some(covfun) +} + +/// Populates the mapping region tables in the current function's covfun record. +fn fill_region_tables<'tcx>( + tcx: TyCtxt<'tcx>, + global_file_table: &GlobalFileTable, + function_coverage: &FunctionCoverage<'tcx>, + covfun: &mut CovfunRecord<'tcx>, +) { + let counter_regions = function_coverage.counter_regions(); + if counter_regions.is_empty() { + return; + } + + // Currently a function's mappings must all be in the same file as its body span. + let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span); + + // Look up the global file ID for that filename. + let global_file_id = global_file_table.global_file_id_for_file_name(file_name); + + // Associate that global file ID with a local file ID for this function. + let local_file_id = covfun.virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'"); + + let ffi::Regions { code_regions, branch_regions, mcdc_branch_regions, mcdc_decision_regions } = + &mut covfun.regions; + + // For each counter/region pair in this function+file, convert it to a + // form suitable for FFI. + for (mapping_kind, region) in counter_regions { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + let span = ffi::CoverageSpan::from_source_region(local_file_id, region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), + }); + } + } + } +} + +/// Generates the contents of the covfun record for this function, which +/// contains the function's coverage mapping data. The record is then stored +/// as a global variable in the `__llvm_covfun` section. +pub(crate) fn generate_covfun_record<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + filenames_ref: u64, + covfun: &CovfunRecord<'tcx>, +) { + let &CovfunRecord { + mangled_function_name, + source_hash, + is_used, + ref virtual_file_mapping, + ref expressions, + ref regions, + } = covfun; + + // Encode the function's coverage mappings into a buffer. + let coverage_mapping_buffer = llvm_cov::write_function_mappings_to_buffer( + &virtual_file_mapping.to_vec(), + expressions, + regions, + ); + + // Concatenate the encoded coverage mappings + let coverage_mapping_size = coverage_mapping_buffer.len(); + let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); + + let func_name_hash = llvm_cov::hash_bytes(mangled_function_name.as_bytes()); + let func_name_hash_val = cx.const_u64(func_name_hash); + let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); + let source_hash_val = cx.const_u64(source_hash); + let filenames_ref_val = cx.const_u64(filenames_ref); + let func_record_val = cx.const_struct( + &[ + func_name_hash_val, + coverage_mapping_size_val, + source_hash_val, + filenames_ref_val, + coverage_mapping_val, + ], + /*packed=*/ true, + ); + + // Choose a variable name to hold this function's covfun data. + // Functions that are used have a suffix ("u") to distinguish them from + // unused copies of the same function (from different CGUs), so that if a + // linker sees both it won't discard the used copy's data. + let func_record_var_name = + CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) + .unwrap(); + debug!("function record var name: {:?}", func_record_var_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, cx.covfun_section_name()); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + if cx.target_spec().supports_comdat() { + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + } + cx.add_used_global(llglobal); +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index c2fcb33f98b..82b6677e203 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,7 +13,7 @@ use tracing::{debug, instrument}; use crate::builder::Builder; use crate::common::CodegenCx; -use crate::coverageinfo::map_data::FunctionCoverageCollector; +use crate::coverageinfo::map_data::FunctionCoverage; use crate::llvm; pub(crate) mod ffi; @@ -24,8 +24,7 @@ mod mapgen; /// Extra per-CGU context/state needed for coverage instrumentation. pub(crate) struct CguCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: - RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, + pub(crate) function_coverage_map: RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, @@ -42,9 +41,7 @@ impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { } } - fn take_function_coverage_map( - &self, - ) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { + fn take_function_coverage_map(&self) -> FxIndexMap<Instance<'tcx>, FunctionCoverage<'tcx>> { self.function_coverage_map.replace(FxIndexMap::default()) } @@ -161,8 +158,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { // This includes functions that were not partitioned into this CGU, // but were MIR-inlined into one of this CGU's functions. coverage_cx.function_coverage_map.borrow_mut().entry(instance).or_insert_with(|| { - FunctionCoverageCollector::new( - instance, + FunctionCoverage::new_used( function_coverage_info, bx.tcx.coverage_ids_info(instance.def), ) diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index a4cb5a25d1b..11043b664f5 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -151,7 +151,7 @@ impl InlineAsmDiagnostic { unsafe { SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)) }; InlineAsmDiagnostic { level: smdiag.level, - cookie: cookie.into(), + cookie, message: smdiag.message, source: smdiag.source, } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index b1ace0033ba..d62632d1431 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -17,7 +17,9 @@ use super::debuginfo::{ DebugEmissionKind, DebugNameTableKind, }; -pub type Bool = c_uint; +/// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, +/// which has a different ABI from Rust or C++ `bool`. +pub type Bool = c_int; pub const True: Bool = 1 as Bool; pub const False: Bool = 0 as Bool; @@ -2317,7 +2319,7 @@ unsafe extern "C" { pub fn LLVMRustGetSMDiagnostic<'a>( DI: &'a DiagnosticInfo, - cookie_out: &mut c_uint, + cookie_out: &mut u64, ) -> &'a SMDiagnostic; pub fn LLVMRustUnpackSMDiagnostic( diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index db2b03d9aed..07eb89e6041 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -230,6 +230,8 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea "aarch64" } else if sess.target.arch == "sparc64" { "sparc" + } else if sess.target.arch == "powerpc64" { + "powerpc" } else { &*sess.target.arch }; @@ -289,6 +291,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), ("sparc", "v8plus") if get_version().0 < 19 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), (_, s) => Some(LLVMFeature::new(s)), } } |