diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs | 100 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs | 8 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 187 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 99 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs | 60 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 29 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 54 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 25 |
12 files changed, 330 insertions, 262 deletions
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 751b2235dc8..ac76b781218 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1574,6 +1574,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { cfi::typeid_for_fnabi(self.tcx, fn_abi, options) }; let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap(); + let dbg_loc = self.get_dbg_loc(); // Test whether the function pointer is associated with the type identifier. let cond = self.type_test(llfn, typeid_metadata); @@ -1582,10 +1583,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { self.cond_br(cond, bb_pass, bb_fail); self.switch_to_block(bb_fail); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } self.abort(); self.unreachable(); self.switch_to_block(bb_pass); + if let Some(dbg_loc) = dbg_loc { + self.set_dbg_loc(dbg_loc); + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index feac97f3e2b..a6e07ea2a60 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,5 +1,7 @@ use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; +use crate::coverageinfo::mapgen::LocalFileId; + /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] @@ -137,8 +139,12 @@ pub(crate) struct CoverageSpan { } impl CoverageSpan { - pub(crate) fn from_source_region(file_id: u32, code_region: &SourceRegion) -> Self { - let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + pub(crate) fn from_source_region( + local_file_id: LocalFileId, + code_region: &SourceRegion, + ) -> Self { + let file_id = local_file_id.as_u32(); + let &SourceRegion { start_line, start_col, end_line, end_col } = code_region; // Internally, LLVM uses the high bit of `end_col` to distinguish between // code regions and gap regions, so it can't be used by the column number. assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs new file mode 100644 index 00000000000..99c2d12b261 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -0,0 +1,100 @@ +//! Safe wrappers for coverage-specific FFI functions. + +use std::ffi::CString; + +use crate::common::AsCCharPtr; +use crate::coverageinfo::ffi; +use crate::llvm; + +pub(crate) fn covmap_var_name() -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovmapVarNameToString(s); + })) + .expect("covmap variable name should not contain NUL") +} + +pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s); + })) + .expect("covmap section name should not contain NUL") +} + +pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s); + })) + .expect("covfun section name should not contain NUL") +} + +pub(crate) fn create_pgo_func_name_var<'ll>( + llfn: &'ll llvm::Value, + mangled_fn_name: &str, +) -> &'ll llvm::Value { + unsafe { + llvm::LLVMRustCoverageCreatePGOFuncNameVar( + llfn, + mangled_fn_name.as_c_char_ptr(), + mangled_fn_name.len(), + ) + } +} + +pub(crate) fn write_filenames_to_buffer<'a>( + filenames: impl IntoIterator<Item = &'a str>, +) -> Vec<u8> { + let (pointers, lengths) = filenames + .into_iter() + .map(|s: &str| (s.as_c_char_ptr(), s.len())) + .unzip::<_, _, Vec<_>, Vec<_>>(); + + llvm::build_byte_buffer(|buffer| unsafe { + llvm::LLVMRustCoverageWriteFilenamesToBuffer( + pointers.as_ptr(), + pointers.len(), + lengths.as_ptr(), + lengths.len(), + buffer, + ); + }) +} + +pub(crate) fn write_function_mappings_to_buffer( + virtual_file_mapping: &[u32], + expressions: &[ffi::CounterExpression], + code_regions: &[ffi::CodeRegion], + branch_regions: &[ffi::BranchRegion], + mcdc_branch_regions: &[ffi::MCDCBranchRegion], + mcdc_decision_regions: &[ffi::MCDCDecisionRegion], +) -> Vec<u8> { + llvm::build_byte_buffer(|buffer| unsafe { + llvm::LLVMRustCoverageWriteFunctionMappingsToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len(), + expressions.as_ptr(), + expressions.len(), + code_regions.as_ptr(), + code_regions.len(), + branch_regions.as_ptr(), + branch_regions.len(), + mcdc_branch_regions.as_ptr(), + mcdc_branch_regions.len(), + mcdc_decision_regions.as_ptr(), + mcdc_decision_regions.len(), + buffer, + ) + }) +} + +/// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`, +/// as required for parts of the LLVM coverage mapping format. +pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { + unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) } +} + +/// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h) +/// as a raw numeric value. For historical reasons, the numeric value is 1 less +/// than the number in the version's name, so `Version7` is actually `6u32`. +pub(crate) fn mapping_version() -> u32 { + unsafe { llvm::LLVMRustCoverageMappingVersion() } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index 5ed640b840e..e3c0df27883 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -6,7 +6,6 @@ use rustc_middle::mir::coverage::{ SourceRegion, }; use rustc_middle::ty::Instance; -use rustc_span::Symbol; use tracing::{debug, instrument}; use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; @@ -180,7 +179,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> { } pub(crate) struct FunctionCoverage<'tcx> { - function_coverage_info: &'tcx FunctionCoverageInfo, + pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo, is_used: bool, counters_seen: BitSet<CounterId>, @@ -199,11 +198,6 @@ impl<'tcx> FunctionCoverage<'tcx> { if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } } - /// Returns an iterator over all filenames used by this function's mappings. - pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> { - self.function_coverage_info.mappings.iter().map(|mapping| mapping.source_region.file_name) - } - /// Convert this function's coverage expression data into a form that can be /// passed through FFI to LLVM. pub(crate) fn counter_expressions( diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index f6378199fe2..b582dd967a7 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,4 +1,5 @@ use std::ffi::CString; +use std::iter; use itertools::Itertools as _; use rustc_abi::Align; @@ -11,15 +12,17 @@ use rustc_index::IndexVec; use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; -use rustc_span::Symbol; +use rustc_session::RemapFileNameExt; +use rustc_session::config::RemapPathScopeComponents; use rustc_span::def_id::DefIdSet; +use rustc_span::{Span, Symbol}; use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::ffi; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; -use crate::{coverageinfo, llvm}; +use crate::coverageinfo::{ffi, llvm_cov}; +use crate::llvm; /// Generates and exports the coverage map, which is embedded in special /// linker sections in the final binary. @@ -33,7 +36,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { // agrees with our Rust-side code. Expected versions (encoded as n-1) are: // - `CovMapVersion::Version7` (6) used by LLVM 18-19 let covmap_version = { - let llvm_covmap_version = coverageinfo::mapping_version(); + let llvm_covmap_version = llvm_cov::mapping_version(); let expected_versions = 6..=6; assert!( expected_versions.contains(&llvm_covmap_version), @@ -69,8 +72,10 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) .collect::<Vec<_>>(); - let all_file_names = - function_coverage_entries.iter().flat_map(|(_, fn_cov)| fn_cov.all_file_names()); + let all_file_names = function_coverage_entries + .iter() + .map(|(_, fn_cov)| fn_cov.function_coverage_info.body_span) + .map(|span| span_file_name(tcx, span)); let global_file_table = GlobalFileTable::new(all_file_names); // Encode all filenames referenced by coverage mappings in this CGU. @@ -78,7 +83,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let filenames_size = filenames_buffer.len(); let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); + let filenames_ref = llvm_cov::hash_bytes(&filenames_buffer); // Generate the coverage map header, which contains the filenames used by // this CGU's coverage mappings, and store it in a well-known global. @@ -95,7 +100,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { let is_used = function_coverage.is_used(); let coverage_mapping_buffer = - encode_mappings_for_function(&global_file_table, &function_coverage); + encode_mappings_for_function(tcx, &global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { if function_coverage.is_used() { @@ -163,13 +168,13 @@ impl GlobalFileTable { Self { raw_file_table } } - fn global_file_id_for_file_name(&self, file_name: Symbol) -> u32 { + fn global_file_id_for_file_name(&self, file_name: Symbol) -> GlobalFileId { let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| { bug!("file name not found in prepared global file table: {file_name}"); }); // The raw file table doesn't include an entry for the working dir // (which has ID 0), so add 1 to get the correct ID. - (raw_id + 1) as u32 + GlobalFileId::from_usize(raw_id + 1) } fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { @@ -187,30 +192,35 @@ impl GlobalFileTable { .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) .to_string_lossy(); - llvm::build_byte_buffer(|buffer| { - coverageinfo::write_filenames_section_to_buffer( - // Insert the working dir at index 0, before the other filenames. - std::iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)), - buffer, - ); - }) + // Insert the working dir at index 0, before the other filenames. + let filenames = + iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)); + llvm_cov::write_filenames_to_buffer(filenames) } } rustc_index::newtype_index! { - struct LocalFileId {} + /// An index into the CGU's overall list of file paths. The underlying paths + /// will be embedded in the `__llvm_covmap` linker section. + struct GlobalFileId {} +} +rustc_index::newtype_index! { + /// An index into a function's list of global file IDs. That underlying list + /// of local-to-global mappings will be embedded in the function's record in + /// the `__llvm_covfun` linker section. + pub(crate) struct LocalFileId {} } /// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) /// file IDs. #[derive(Default)] struct VirtualFileMapping { - local_to_global: IndexVec<LocalFileId, u32>, - global_to_local: FxIndexMap<u32, LocalFileId>, + local_to_global: IndexVec<LocalFileId, GlobalFileId>, + global_to_local: FxIndexMap<GlobalFileId, LocalFileId>, } impl VirtualFileMapping { - fn local_id_for_global(&mut self, global_file_id: u32) -> LocalFileId { + fn local_id_for_global(&mut self, global_file_id: GlobalFileId) -> LocalFileId { *self .global_to_local .entry(global_file_id) @@ -218,16 +228,26 @@ impl VirtualFileMapping { } fn into_vec(self) -> Vec<u32> { - self.local_to_global.raw + // This conversion should be optimized away to ~zero overhead. + // In any case, it's probably not hot enough to worry about. + self.local_to_global.into_iter().map(|global| global.as_u32()).collect() } } +fn span_file_name(tcx: TyCtxt<'_>, span: Span) -> Symbol { + let source_file = tcx.sess.source_map().lookup_source_file(span.lo()); + let name = + source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(); + Symbol::intern(&name) +} + /// Using the expressions and counter regions collected for a single function, /// generate the variable-sized payload of its corresponding `__llvm_covfun` /// entry. The payload is returned as a vector of bytes. /// /// Newly-encountered filenames will be added to the global file table. fn encode_mappings_for_function( + tcx: TyCtxt<'_>, global_file_table: &GlobalFileTable, function_coverage: &FunctionCoverage<'_>, ) -> Vec<u8> { @@ -244,69 +264,58 @@ fn encode_mappings_for_function( let mut mcdc_branch_regions = vec![]; let mut mcdc_decision_regions = vec![]; - // Group mappings into runs with the same filename, preserving the order - // yielded by `FunctionCoverage`. - // Prepare file IDs for each filename, and prepare the mapping data so that - // we can pass it through FFI to LLVM. - for (file_name, counter_regions_for_file) in - &counter_regions.group_by(|(_, region)| region.file_name) - { - // Look up the global file ID for this filename. - let global_file_id = global_file_table.global_file_id_for_file_name(file_name); - - // Associate that global file ID with a local file ID for this function. - let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); - debug!(" file id: {local_file_id:?} => global {global_file_id} = '{file_name:?}'"); - - // For each counter/region pair in this function+file, convert it to a - // form suitable for FFI. - for (mapping_kind, region) in counter_regions_for_file { - debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - let span = ffi::CoverageSpan::from_source_region(local_file_id.as_u32(), region); - match mapping_kind { - MappingKind::Code(term) => { - code_regions - .push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); - } - MappingKind::Branch { true_term, false_term } => { - branch_regions.push(ffi::BranchRegion { - span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), - }); - } - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - mcdc_branch_regions.push(ffi::MCDCBranchRegion { - span, - true_counter: ffi::Counter::from_term(true_term), - false_counter: ffi::Counter::from_term(false_term), - mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), - }); - } - MappingKind::MCDCDecision(mcdc_decision_params) => { - mcdc_decision_regions.push(ffi::MCDCDecisionRegion { - span, - mcdc_decision_params: ffi::mcdc::DecisionParameters::from( - mcdc_decision_params, - ), - }); - } + // Currently a function's mappings must all be in the same file as its body span. + let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span); + + // Look up the global file ID for that filename. + let global_file_id = global_file_table.global_file_id_for_file_name(file_name); + + // Associate that global file ID with a local file ID for this function. + let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'"); + + // For each counter/region pair in this function+file, convert it to a + // form suitable for FFI. + for (mapping_kind, region) in counter_regions { + debug!("Adding counter {mapping_kind:?} to map for {region:?}"); + let span = ffi::CoverageSpan::from_source_region(local_file_id, region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), + }); } } } // Encode the function's coverage mappings into a buffer. - llvm::build_byte_buffer(|buffer| { - coverageinfo::write_mapping_to_buffer( - virtual_file_mapping.into_vec(), - expressions, - &code_regions, - &branch_regions, - &mcdc_branch_regions, - &mcdc_decision_regions, - buffer, - ); - }) + llvm_cov::write_function_mappings_to_buffer( + &virtual_file_mapping.into_vec(), + &expressions, + &code_regions, + &branch_regions, + &mcdc_branch_regions, + &mcdc_decision_regions, + ) } /// Generates the contents of the covmap record for this CGU, which mostly @@ -335,23 +344,11 @@ fn generate_covmap_record<'ll>( let covmap_data = cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false); - let covmap_var_name = CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - })) - .unwrap(); - debug!("covmap var name: {:?}", covmap_var_name); - - let covmap_section_name = CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - })) - .expect("covmap section name should not contain NUL"); - debug!("covmap section name: {:?}", covmap_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(covmap_data), &covmap_var_name); + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(covmap_data), &llvm_cov::covmap_var_name()); llvm::set_initializer(llglobal, covmap_data); llvm::set_global_constant(llglobal, true); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::set_section(llglobal, &covmap_section_name); + llvm::set_section(llglobal, &llvm_cov::covmap_section_name(cx.llmod)); // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. // <https://llvm.org/docs/CoverageMappingFormat.html> llvm::set_alignment(llglobal, Align::EIGHT); @@ -373,7 +370,7 @@ fn generate_covfun_record( let coverage_mapping_size = coverage_mapping_buffer.len(); let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - let func_name_hash = coverageinfo::hash_bytes(mangled_function_name.as_bytes()); + let func_name_hash = llvm_cov::hash_bytes(mangled_function_name.as_bytes()); let func_name_hash_val = cx.const_u64(func_name_hash); let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); let source_hash_val = cx.const_u64(source_hash); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index aaba0684c12..bf773cd2667 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,24 +1,23 @@ use std::cell::{OnceCell, RefCell}; use std::ffi::{CStr, CString}; -use libc::c_uint; use rustc_abi::Size; use rustc_codegen_ssa::traits::{ BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; -use rustc_llvm::RustString; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use rustc_middle::ty::layout::HasTyCtxt; use tracing::{debug, instrument}; use crate::builder::Builder; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::coverageinfo::map_data::FunctionCoverageCollector; use crate::llvm; pub(crate) mod ffi; +mod llvm_cov; pub(crate) mod map_data; mod mapgen; @@ -80,12 +79,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) fn covfun_section_name(&self) -> &CStr { - self.coverage_cx().covfun_section_name.get_or_init(|| { - CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(self.llmod, s); - })) - .expect("covfun section name should not contain NUL") - }) + self.coverage_cx() + .covfun_section_name + .get_or_init(|| llvm_cov::covfun_section_name(self.llmod)) } /// For LLVM codegen, returns a function-specific `Value` for a global @@ -95,9 +91,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { debug!("getting pgo_func_name_var for instance={:?}", instance); let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut(); - pgo_func_name_var_map - .entry(instance) - .or_insert_with(|| create_pgo_func_name_var(self, instance)) + pgo_func_name_var_map.entry(instance).or_insert_with(|| { + let llfn = self.get_fn(instance); + let mangled_fn_name: &str = self.tcx.symbol_name(instance).name; + llvm_cov::create_pgo_func_name_var(llfn, mangled_fn_name) + }) } } @@ -225,80 +223,3 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } } - -/// Calls llvm::createPGOFuncNameVar() with the given function instance's -/// mangled function name. The LLVM API returns an llvm::GlobalVariable -/// containing the function name, with the specific variable name and linkage -/// required by LLVM InstrProf source-based coverage instrumentation. Use -/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per -/// `Instance`. -fn create_pgo_func_name_var<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - instance: Instance<'tcx>, -) -> &'ll llvm::Value { - let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name; - let llfn = cx.get_fn(instance); - unsafe { - llvm::LLVMRustCoverageCreatePGOFuncNameVar( - llfn, - mangled_fn_name.as_c_char_ptr(), - mangled_fn_name.len(), - ) - } -} - -pub(crate) fn write_filenames_section_to_buffer<'a>( - filenames: impl IntoIterator<Item = &'a str>, - buffer: &RustString, -) { - let (pointers, lengths) = filenames - .into_iter() - .map(|s: &str| (s.as_c_char_ptr(), s.len())) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - unsafe { - llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( - pointers.as_ptr(), - pointers.len(), - lengths.as_ptr(), - lengths.len(), - buffer, - ); - } -} - -pub(crate) fn write_mapping_to_buffer( - virtual_file_mapping: Vec<u32>, - expressions: Vec<ffi::CounterExpression>, - code_regions: &[ffi::CodeRegion], - branch_regions: &[ffi::BranchRegion], - mcdc_branch_regions: &[ffi::MCDCBranchRegion], - mcdc_decision_regions: &[ffi::MCDCDecisionRegion], - buffer: &RustString, -) { - unsafe { - llvm::LLVMRustCoverageWriteMappingToBuffer( - virtual_file_mapping.as_ptr(), - virtual_file_mapping.len() as c_uint, - expressions.as_ptr(), - expressions.len() as c_uint, - code_regions.as_ptr(), - code_regions.len() as c_uint, - branch_regions.as_ptr(), - branch_regions.len() as c_uint, - mcdc_branch_regions.as_ptr(), - mcdc_branch_regions.len() as c_uint, - mcdc_decision_regions.as_ptr(), - mcdc_decision_regions.len() as c_uint, - buffer, - ); - } -} - -pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_c_char_ptr(), bytes.len()) } -} - -pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index ac6c2fb1b83..0f1909486ec 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -1,11 +1,15 @@ +use std::collections::hash_map::Entry; + use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; use rustc_index::Idx; use rustc_index::bit_set::BitSet; use rustc_middle::mir::{Body, SourceScope}; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::{self, Instance}; use rustc_session::config::DebugInfo; +use rustc_span::BytePos; use super::metadata::file_metadata; use super::utils::DIB; @@ -37,10 +41,20 @@ pub(crate) fn compute_mir_scopes<'ll, 'tcx>( None }; let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); + let mut discriminators = FxHashMap::default(); // Instantiate all scopes. for idx in 0..mir.source_scopes.len() { let scope = SourceScope::new(idx); - make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); + make_mir_scope( + cx, + instance, + mir, + &variables, + debug_context, + &mut instantiated, + &mut discriminators, + scope, + ); } assert!(instantiated.count() == mir.source_scopes.len()); } @@ -52,6 +66,7 @@ fn make_mir_scope<'ll, 'tcx>( variables: &Option<BitSet<SourceScope>>, debug_context: &mut FunctionDebugContext<'tcx, &'ll DIScope, &'ll DILocation>, instantiated: &mut BitSet<SourceScope>, + discriminators: &mut FxHashMap<BytePos, u32>, scope: SourceScope, ) { if instantiated.contains(scope) { @@ -60,7 +75,16 @@ fn make_mir_scope<'ll, 'tcx>( let scope_data = &mir.source_scopes[scope]; let parent_scope = if let Some(parent) = scope_data.parent_scope { - make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); + make_mir_scope( + cx, + instance, + mir, + variables, + debug_context, + instantiated, + discriminators, + parent, + ); debug_context.scopes[parent] } else { // The root is the function itself. @@ -117,7 +141,37 @@ fn make_mir_scope<'ll, 'tcx>( // FIXME(eddyb) this doesn't account for the macro-related // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); - cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) + let loc = cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span); + + // NB: In order to produce proper debug info for variables (particularly + // arguments) in multiply-inline functions, LLVM expects to see a single + // DILocalVariable with multiple different DILocations in the IR. While + // the source information for each DILocation would be identical, their + // inlinedAt attributes will be unique to the particular callsite. + // + // We generate DILocations here based on the callsite's location in the + // source code. A single location in the source code usually can't + // produce multiple distinct calls so this mostly works, until + // proc-macros get involved. A proc-macro can generate multiple calls + // at the same span, which breaks the assumption that we're going to + // produce a unique DILocation for every scope we process here. We + // have to explicitly add discriminators if we see inlines into the + // same source code location. + // + // Note further that we can't key this hashtable on the span itself, + // because these spans could have distinct SyntaxContexts. We have + // to key on exactly what we're giving to LLVM. + match discriminators.entry(callsite_span.lo()) { + Entry::Occupied(mut o) => { + *o.get_mut() += 1; + unsafe { llvm::LLVMRustDILocationCloneWithBaseDiscriminator(loc, *o.get()) } + .expect("Failed to encode discriminator in DILocation") + } + Entry::Vacant(v) => { + v.insert(0); + loc + } + } }); debug_context.scopes[scope] = DebugScope { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 9e1e5127e80..89492e4b9fe 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -206,6 +206,10 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { } } + fn get_dbg_loc(&self) -> Option<&'ll DILocation> { + unsafe { llvm::LLVMGetCurrentDebugLocation2(self.llbuilder) } + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { gdb::insert_reference_to_gdb_debug_scripts_section_global(self) } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index b85d28a2f1f..3dfb86d422d 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -22,7 +22,6 @@ use std::any::Any; use std::ffi::CStr; -use std::io::Write; use std::mem::ManuallyDrop; use back::owned_target_machine::OwnedTargetMachine; @@ -165,30 +164,12 @@ impl WriteBackendMethods for LlvmCodegenBackend { type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { - unsafe { - let mut size = 0; - let cstr = llvm::LLVMRustPrintPassTimings(&raw mut size); - if cstr.is_null() { - println!("failed to get pass timings"); - } else { - let timings = std::slice::from_raw_parts(cstr as *const u8, size); - std::io::stdout().write_all(timings).unwrap(); - libc::free(cstr as *mut _); - } - } + let timings = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintPassTimings(s) }).unwrap(); + print!("{timings}"); } fn print_statistics(&self) { - unsafe { - let mut size = 0; - let cstr = llvm::LLVMRustPrintStatistics(&raw mut size); - if cstr.is_null() { - println!("failed to get pass stats"); - } else { - let stats = std::slice::from_raw_parts(cstr as *const u8, size); - std::io::stdout().write_all(stats).unwrap(); - libc::free(cstr as *mut _); - } - } + let stats = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintStatistics(s) }).unwrap(); + print!("{stats}"); } fn run_link( cgcx: &CodegenContext<Self>, @@ -401,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) + link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs) } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d84ae8d8836..7f59264824e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -204,7 +204,7 @@ pub enum DLLStorageClass { DllExport = 2, // Function to be accessible from DLL. } -/// Matches LLVMRustAttribute in LLVMWrapper.h +/// Must match the layout of `LLVMRustAttributeKind`. /// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, /// though it is not ABI compatible (since it's a C++ enum) #[repr(C)] @@ -1063,6 +1063,7 @@ unsafe extern "C" { // Metadata pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata); + pub fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>; // Terminators pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; @@ -1765,11 +1766,11 @@ unsafe extern "C" { /// Returns a string describing the last error caused by an LLVMRust* call. pub fn LLVMRustGetLastError() -> *const c_char; - /// Print the pass timings since static dtors aren't picking them up. - pub fn LLVMRustPrintPassTimings(size: *const size_t) -> *const c_char; + /// Prints the timing information collected by `-Ztime-llvm-passes`. + pub(crate) fn LLVMRustPrintPassTimings(OutStr: &RustString); - /// Print the statistics since static dtors aren't picking them up. - pub fn LLVMRustPrintStatistics(size: *const size_t) -> *const c_char; + /// Prints the statistics collected by `-Zprint-codegen-stats`. + pub(crate) fn LLVMRustPrintStatistics(OutStr: &RustString); /// Prepares inline assembly. pub fn LLVMRustInlineAsm( @@ -1789,8 +1790,7 @@ unsafe extern "C" { ConstraintsLen: size_t, ) -> bool; - #[allow(improper_ctypes)] - pub(crate) fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, Lengths: *const size_t, @@ -1798,20 +1798,19 @@ unsafe extern "C" { BufferOut: &RustString, ); - #[allow(improper_ctypes)] - pub(crate) fn LLVMRustCoverageWriteMappingToBuffer( + pub(crate) fn LLVMRustCoverageWriteFunctionMappingsToBuffer( VirtualFileMappingIDs: *const c_uint, - NumVirtualFileMappingIDs: c_uint, + NumVirtualFileMappingIDs: size_t, Expressions: *const crate::coverageinfo::ffi::CounterExpression, - NumExpressions: c_uint, + NumExpressions: size_t, CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, - NumCodeRegions: c_uint, + NumCodeRegions: size_t, BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, - NumBranchRegions: c_uint, + NumBranchRegions: size_t, MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, - NumMCDCBranchRegions: c_uint, + NumMCDCBranchRegions: size_t, MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, - NumMCDCDecisionRegions: c_uint, + NumMCDCDecisionRegions: size_t, BufferOut: &RustString, ); @@ -1820,16 +1819,13 @@ unsafe extern "C" { FuncName: *const c_char, FuncNameLen: size_t, ) -> &Value; - pub(crate) fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64; - #[allow(improper_ctypes)] - pub(crate) fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString); - #[allow(improper_ctypes)] - pub(crate) fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString); - #[allow(improper_ctypes)] - pub(crate) fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; @@ -2179,18 +2175,19 @@ unsafe extern "C" { Scope: &'a DIScope, InlinedAt: Option<&'a DILocation>, ) -> &'a DILocation; + pub fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( + Location: &'a DILocation, + BD: c_uint, + ) -> Option<&'a DILocation>; pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; pub fn LLVMRustDIBuilderCreateOpLLVMFragment() -> u64; - #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); - #[allow(improper_ctypes)] pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - #[allow(improper_ctypes)] pub(crate) fn LLVMRustPrintTargetCPUs(TM: &TargetMachine, OutStr: &RustString); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( @@ -2295,10 +2292,8 @@ unsafe extern "C" { pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); - #[allow(improper_ctypes)] pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); - #[allow(improper_ctypes)] pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( DI: &'a DiagnosticInfo, pass_name_out: &RustString, @@ -2316,7 +2311,6 @@ unsafe extern "C" { message_out: &mut Option<&'a Twine>, ); - #[allow(improper_ctypes)] pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; @@ -2325,7 +2319,6 @@ unsafe extern "C" { cookie_out: &mut c_uint, ) -> &'a SMDiagnostic; - #[allow(improper_ctypes)] pub fn LLVMRustUnpackSMDiagnostic( d: &SMDiagnostic, message_out: &RustString, @@ -2372,7 +2365,6 @@ unsafe extern "C" { pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub fn LLVMRustModuleCost(M: &Module) -> u64; - #[allow(improper_ctypes)] pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); pub fn LLVMRustThinLTOBufferCreate( @@ -2425,7 +2417,6 @@ unsafe extern "C" { bytecode_len: usize, ) -> bool; pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); - #[allow(improper_ctypes)] pub fn LLVMRustComputeLTOCacheKey( key_out: &RustString, mod_id: *const c_char, @@ -2448,7 +2439,6 @@ unsafe extern "C" { pgo_available: bool, ); - #[allow(improper_ctypes)] pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 3b0bf47366e..909afe35a17 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -1,6 +1,5 @@ #![allow(non_snake_case)] -use std::cell::RefCell; use std::ffi::{CStr, CString}; use std::ops::Deref; use std::ptr; @@ -301,15 +300,11 @@ pub fn set_value_name(value: &Value, name: &[u8]) { } pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error> { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - String::from_utf8(sr.bytes.into_inner()) + String::from_utf8(RustString::build_byte_buffer(f)) } pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> { - let sr = RustString { bytes: RefCell::new(Vec::new()) }; - f(&sr); - sr.bytes.into_inner() + RustString::build_byte_buffer(f) } pub fn twine_to_string(tr: &Twine) -> String { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 6f2d86cc601..db2b03d9aed 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -228,6 +228,8 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea "x86" } else if sess.target.arch == "arm64ec" { "aarch64" + } else if sess.target.arch == "sparc64" { + "sparc" } else { &*sess.target.arch }; @@ -280,6 +282,13 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea // Support for `wide-arithmetic` will first land in LLVM 20 as part of // llvm/llvm-project#111598 ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". + // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 + // Before LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available". + // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 + ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), + ("sparc", "v8plus") if get_version().0 < 19 => None, (_, s) => Some(LLVMFeature::new(s)), } } @@ -335,15 +344,23 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { }) { if enabled { + // Also add all transitively implied features. features.extend(sess.target.implied_target_features(std::iter::once(feature))); } else { + // Remove transitively reverse-implied features. + // We don't care about the order in `features` since the only thing we use it for is the // `features.contains` below. #[allow(rustc::potential_query_instability)] features.retain(|f| { - // Keep a feature if it does not imply `feature`. Or, equivalently, - // remove the reverse-dependencies of `feature`. - !sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) + if sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) { + // If `f` if implies `feature`, then `!feature` implies `!f`, so we have to + // remove `f`. (This is the standard logical contraposition principle.) + false + } else { + // We can keep `f`. + true + } }); } } @@ -619,6 +636,8 @@ pub(crate) fn global_llvm_features( .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(sess, v).is_some()) + // Drop +v8plus feature introduced in LLVM 20. + .filter(|v| *v != "+v8plus" || get_version() >= (20, 0, 0)) .map(String::from), ); |