diff options
Diffstat (limited to 'src/librustc_codegen_llvm')
| -rw-r--r-- | src/librustc_codegen_llvm/attributes.rs | 3 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/base.rs | 11 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/builder.rs | 2 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/consts.rs | 10 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/coverageinfo/mapgen.rs | 274 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/coverageinfo/mod.rs | 266 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/intrinsic.rs | 102 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/llvm/ffi.rs | 66 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/llvm/mod.rs | 44 |
9 files changed, 666 insertions, 112 deletions
diff --git a/src/librustc_codegen_llvm/attributes.rs b/src/librustc_codegen_llvm/attributes.rs index 89b548a9c5a..227a87ff819 100644 --- a/src/librustc_codegen_llvm/attributes.rs +++ b/src/librustc_codegen_llvm/attributes.rs @@ -133,6 +133,9 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { return; } + // FIXME(richkadel): Make sure probestack plays nice with `-Z instrument-coverage` + // or disable it if not, similar to above early exits. + // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. llvm::AddFunctionAttrStringValue( diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index d5e0d7d36ee..b19199b9cfa 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -144,17 +144,18 @@ pub fn compile_codegen_unit( } } + // Finalize code coverage by injecting the coverage map. Note, the coverage map will + // also be added to the `llvm.used` variable, created next. + if cx.sess().opts.debugging_opts.instrument_coverage { + cx.coverageinfo_finalize(); + } + // Create the llvm.used variable // This variable has type [N x i8*] and is stored in the llvm.metadata section if !cx.used_statics().borrow().is_empty() { cx.create_used_variable() } - // Finalize code coverage by injecting the coverage map - if cx.sess().opts.debugging_opts.instrument_coverage { - cx.coverageinfo_finalize(); - } - // Finalize debuginfo if cx.sess().opts.debuginfo != DebugInfo::None { cx.debuginfo_finalize(); diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs index 6a38323f7ca..d58aad340a1 100644 --- a/src/librustc_codegen_llvm/builder.rs +++ b/src/librustc_codegen_llvm/builder.rs @@ -1060,7 +1060,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_name, hash, num_counters, index ); - let llfn = unsafe { llvm::LLVMRustGetInstrprofIncrementIntrinsic(self.cx().llmod) }; + let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; let args = &[fn_name, hash, num_counters, index]; let args = self.check_call("call", llfn, args); diff --git a/src/librustc_codegen_llvm/consts.rs b/src/librustc_codegen_llvm/consts.rs index 90887b760fb..c954415f19f 100644 --- a/src/librustc_codegen_llvm/consts.rs +++ b/src/librustc_codegen_llvm/consts.rs @@ -493,10 +493,14 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> { } if attrs.flags.contains(CodegenFnAttrFlags::USED) { - // This static will be stored in the llvm.used variable which is an array of i8* - let cast = llvm::LLVMConstPointerCast(g, self.type_i8p()); - self.used_statics.borrow_mut().push(cast); + self.add_used_global(g); } } } + + /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. + fn add_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.used_statics.borrow_mut().push(cast); + } } diff --git a/src/librustc_codegen_llvm/coverageinfo/mapgen.rs b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs new file mode 100644 index 00000000000..7f48b1d864c --- /dev/null +++ b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs @@ -0,0 +1,274 @@ +use crate::llvm; + +use crate::common::CodegenCx; +use crate::coverageinfo; + +use log::debug; +use rustc_codegen_ssa::coverageinfo::map::*; +use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, MiscMethods}; +use rustc_data_structures::fx::FxHashMap; +use rustc_llvm::RustString; +use rustc_middle::ty::Instance; +use rustc_middle::{bug, mir}; + +use std::collections::BTreeMap; +use std::ffi::CString; +use std::path::PathBuf; + +// FIXME(richkadel): Complete all variations of generating and exporting the coverage map to LLVM. +// The current implementation is an initial foundation with basic capabilities (Counters, but not +// CounterExpressions, etc.). + +/// Generates and exports the Coverage Map. +/// +/// This Coverage Map complies with Coverage Mapping Format version 3 (zero-based encoded as 2), +/// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/llvmorg-8.0.0/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format) +/// and published in Rust's current (July 2020) fork of LLVM. This version is supported by the +/// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM. +/// +/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with +/// version 3. Clang's implementation of Coverage Map generation was referenced when implementing +/// this Rust version, and though the format documentation is very explicit and detailed, some +/// undocumented details in Clang's implementation (that may or may not be important) were also +/// replicated for Rust's Coverage Map. +pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + let mut coverage_writer = CoverageMappingWriter::new(cx); + + let function_coverage_map = cx.coverage_context().take_function_coverage_map(); + + // Encode coverage mappings and generate function records + let mut function_records = Vec::<&'ll llvm::Value>::new(); + let coverage_mappings_buffer = llvm::build_byte_buffer(|coverage_mappings_buffer| { + for (instance, function_coverage) in function_coverage_map.into_iter() { + if let Some(function_record) = coverage_writer.write_function_mappings_and_record( + instance, + function_coverage, + coverage_mappings_buffer, + ) { + function_records.push(function_record); + } + } + }); + + // Encode all filenames covered in this module, ordered by `file_id` + let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { + coverageinfo::write_filenames_section_to_buffer( + &coverage_writer.filenames, + filenames_buffer, + ); + }); + + if coverage_mappings_buffer.len() > 0 { + // Generate the LLVM IR representation of the coverage map and store it in a well-known + // global constant. + coverage_writer.write_coverage_map( + function_records, + filenames_buffer, + coverage_mappings_buffer, + ); + } +} + +struct CoverageMappingWriter<'a, 'll, 'tcx> { + cx: &'a CodegenCx<'ll, 'tcx>, + filenames: Vec<CString>, + filename_to_index: FxHashMap<CString, u32>, +} + +impl<'a, 'll, 'tcx> CoverageMappingWriter<'a, 'll, 'tcx> { + fn new(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { + Self { cx, filenames: Vec::new(), filename_to_index: FxHashMap::<CString, u32>::default() } + } + + /// For the given function, get the coverage region data, stream it to the given buffer, and + /// then generate and return a new function record. + fn write_function_mappings_and_record( + &mut self, + instance: Instance<'tcx>, + mut function_coverage: FunctionCoverage, + coverage_mappings_buffer: &RustString, + ) -> Option<&'ll llvm::Value> { + let cx = self.cx; + let coverageinfo: &mir::CoverageInfo = cx.tcx.coverageinfo(instance.def_id()); + debug!( + "Generate coverage map for: {:?}, num_counters: {}, num_expressions: {}", + instance, coverageinfo.num_counters, coverageinfo.num_expressions + ); + debug_assert!(coverageinfo.num_counters > 0); + + let regions_in_file_order = function_coverage.regions_in_file_order(cx.sess().source_map()); + if regions_in_file_order.len() == 0 { + return None; + } + + // Stream the coverage mapping regions for the function (`instance`) to the buffer, and + // compute the data byte size used. + let old_len = coverage_mappings_buffer.len(); + self.regions_to_mappings(regions_in_file_order, coverage_mappings_buffer); + let mapping_data_size = coverage_mappings_buffer.len() - old_len; + debug_assert!(mapping_data_size > 0); + + let mangled_function_name = cx.tcx.symbol_name(instance).to_string(); + let name_ref = coverageinfo::compute_hash(&mangled_function_name); + let function_source_hash = function_coverage.source_hash(); + + // Generate and return the function record + let name_ref_val = cx.const_u64(name_ref); + let mapping_data_size_val = cx.const_u32(mapping_data_size as u32); + let func_hash_val = cx.const_u64(function_source_hash); + Some(cx.const_struct( + &[name_ref_val, mapping_data_size_val, func_hash_val], + /*packed=*/ true, + )) + } + + /// For each coverage region, extract its coverage data from the earlier coverage analysis. + /// Use LLVM APIs to convert the data into buffered bytes compliant with the LLVM Coverage + /// Mapping format. + fn regions_to_mappings( + &mut self, + regions_in_file_order: BTreeMap<PathBuf, BTreeMap<CoverageLoc, (usize, CoverageKind)>>, + coverage_mappings_buffer: &RustString, + ) { + let mut virtual_file_mapping = Vec::new(); + let mut mapping_regions = coverageinfo::SmallVectorCounterMappingRegion::new(); + let mut expressions = coverageinfo::SmallVectorCounterExpression::new(); + + for (file_id, (file_path, file_coverage_regions)) in + regions_in_file_order.into_iter().enumerate() + { + let file_id = file_id as u32; + let filename = CString::new(file_path.to_string_lossy().to_string()) + .expect("null error converting filename to C string"); + debug!(" file_id: {} = '{:?}'", file_id, filename); + let filenames_index = match self.filename_to_index.get(&filename) { + Some(index) => *index, + None => { + let index = self.filenames.len() as u32; + self.filenames.push(filename.clone()); + self.filename_to_index.insert(filename, index); + index + } + }; + virtual_file_mapping.push(filenames_index); + + let mut mapping_indexes = vec![0 as u32; file_coverage_regions.len()]; + for (mapping_index, (region_id, _)) in file_coverage_regions.values().enumerate() { + mapping_indexes[*region_id] = mapping_index as u32; + } + + for (region_loc, (region_id, region_kind)) in file_coverage_regions.into_iter() { + let mapping_index = mapping_indexes[region_id]; + match region_kind { + CoverageKind::Counter => { + debug!( + " Counter {}, file_id: {}, region_loc: {}", + mapping_index, file_id, region_loc + ); + mapping_regions.push_from( + mapping_index, + file_id, + region_loc.start_line, + region_loc.start_col, + region_loc.end_line, + region_loc.end_col, + ); + } + CoverageKind::CounterExpression(lhs, op, rhs) => { + debug!( + " CounterExpression {} = {} {:?} {}, file_id: {}, region_loc: {:?}", + mapping_index, lhs, op, rhs, file_id, region_loc, + ); + mapping_regions.push_from( + mapping_index, + file_id, + region_loc.start_line, + region_loc.start_col, + region_loc.end_line, + region_loc.end_col, + ); + expressions.push_from(op, lhs, rhs); + } + CoverageKind::Unreachable => { + debug!( + " Unreachable region, file_id: {}, region_loc: {:?}", + file_id, region_loc, + ); + bug!("Unreachable region not expected and not yet handled!") + // FIXME(richkadel): implement and call + // mapping_regions.push_from(...) for unreachable regions + } + } + } + } + + // Encode and append the current function's coverage mapping data + coverageinfo::write_mapping_to_buffer( + virtual_file_mapping, + expressions, + mapping_regions, + coverage_mappings_buffer, + ); + } + + fn write_coverage_map( + self, + function_records: Vec<&'ll llvm::Value>, + filenames_buffer: Vec<u8>, + mut coverage_mappings_buffer: Vec<u8>, + ) { + let cx = self.cx; + + // Concatenate the encoded filenames and encoded coverage mappings, and add additional zero + // bytes as-needed to ensure 8-byte alignment. + let mut coverage_size = coverage_mappings_buffer.len(); + let filenames_size = filenames_buffer.len(); + let remaining_bytes = + (filenames_size + coverage_size) % coverageinfo::COVMAP_VAR_ALIGN_BYTES; + if remaining_bytes > 0 { + let pad = coverageinfo::COVMAP_VAR_ALIGN_BYTES - remaining_bytes; + coverage_mappings_buffer.append(&mut [0].repeat(pad)); + coverage_size += pad; + } + let filenames_and_coverage_mappings = [filenames_buffer, coverage_mappings_buffer].concat(); + let filenames_and_coverage_mappings_val = + cx.const_bytes(&filenames_and_coverage_mappings[..]); + + debug!( + "cov map: n_records = {}, filenames_size = {}, coverage_size = {}, 0-based version = {}", + function_records.len(), + filenames_size, + coverage_size, + coverageinfo::mapping_version() + ); + + // Create the coverage data header + let n_records_val = cx.const_u32(function_records.len() as u32); + let filenames_size_val = cx.const_u32(filenames_size as u32); + let coverage_size_val = cx.const_u32(coverage_size as u32); + let version_val = cx.const_u32(coverageinfo::mapping_version()); + let cov_data_header_val = cx.const_struct( + &[n_records_val, filenames_size_val, coverage_size_val, version_val], + /*packed=*/ false, + ); + + // Create the function records array + let name_ref_from_u64 = cx.type_i64(); + let mapping_data_size_from_u32 = cx.type_i32(); + let func_hash_from_u64 = cx.type_i64(); + let function_record_ty = cx.type_struct( + &[name_ref_from_u64, mapping_data_size_from_u32, func_hash_from_u64], + /*packed=*/ true, + ); + let function_records_val = cx.const_array(function_record_ty, &function_records[..]); + + // Create the complete LLVM coverage data value to add to the LLVM IR + let cov_data_val = cx.const_struct( + &[cov_data_header_val, function_records_val, filenames_and_coverage_mappings_val], + /*packed=*/ false, + ); + + // Save the coverage data value to LLVM IR + coverageinfo::save_map_to_mod(cx, cov_data_val); + } +} diff --git a/src/librustc_codegen_llvm/coverageinfo/mod.rs b/src/librustc_codegen_llvm/coverageinfo/mod.rs index ff9f8f7aeaa..76894bcd6c1 100644 --- a/src/librustc_codegen_llvm/coverageinfo/mod.rs +++ b/src/librustc_codegen_llvm/coverageinfo/mod.rs @@ -1,67 +1,44 @@ +use crate::llvm; + use crate::builder::Builder; use crate::common::CodegenCx; + +use libc::c_uint; use log::debug; use rustc_codegen_ssa::coverageinfo::map::*; -use rustc_codegen_ssa::traits::{CoverageInfoBuilderMethods, CoverageInfoMethods}; +use rustc_codegen_ssa::traits::{ + BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, StaticMethods, +}; use rustc_data_structures::fx::FxHashMap; +use rustc_llvm::RustString; use rustc_middle::ty::Instance; use std::cell::RefCell; +use std::ffi::CString; + +pub mod mapgen; + +const COVMAP_VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'tcx> { // Coverage region data for each instrumented function identified by DefId. - pub(crate) coverage_regions: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageRegions>>, + pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage>>, } impl<'tcx> CrateCoverageContext<'tcx> { pub fn new() -> Self { - Self { coverage_regions: Default::default() } + Self { function_coverage_map: Default::default() } } -} -/// Generates and exports the Coverage Map. -// FIXME(richkadel): Actually generate and export the coverage map to LLVM. -// The current implementation is actually just debug messages to show the data is available. -pub fn finalize(cx: &CodegenCx<'_, '_>) { - let coverage_regions = &*cx.coverage_context().coverage_regions.borrow(); - for instance in coverage_regions.keys() { - let coverageinfo = cx.tcx.coverageinfo(instance.def_id()); - debug_assert!(coverageinfo.num_counters > 0); - debug!( - "Generate coverage map for: {:?}, hash: {}, num_counters: {}", - instance, coverageinfo.hash, coverageinfo.num_counters - ); - let function_coverage_regions = &coverage_regions[instance]; - for (index, region) in function_coverage_regions.indexed_regions() { - match region.kind { - CoverageKind::Counter => debug!( - " Counter {}, for {}..{}", - index, region.coverage_span.start_byte_pos, region.coverage_span.end_byte_pos - ), - CoverageKind::CounterExpression(lhs, op, rhs) => debug!( - " CounterExpression {} = {} {:?} {}, for {}..{}", - index, - lhs, - op, - rhs, - region.coverage_span.start_byte_pos, - region.coverage_span.end_byte_pos - ), - } - } - for unreachable in function_coverage_regions.unreachable_regions() { - debug!( - " Unreachable code region: {}..{}", - unreachable.start_byte_pos, unreachable.end_byte_pos - ); - } + pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage> { + self.function_coverage_map.replace(FxHashMap::default()) } } impl CoverageInfoMethods for CodegenCx<'ll, 'tcx> { fn coverageinfo_finalize(&self) { - finalize(self) + mapgen::finalize(self) } } @@ -69,20 +46,22 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { fn add_counter_region( &mut self, instance: Instance<'tcx>, + function_source_hash: u64, index: u32, start_byte_pos: u32, end_byte_pos: u32, ) { debug!( - "adding counter to coverage map: instance={:?}, index={}, byte range {}..{}", - instance, index, start_byte_pos, end_byte_pos, - ); - let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut(); - coverage_regions.entry(instance).or_default().add_counter( - index, - start_byte_pos, - end_byte_pos, + "adding counter to coverage_regions: instance={:?}, function_source_hash={}, index={}, byte range {}..{}", + instance, function_source_hash, index, start_byte_pos, end_byte_pos, ); + let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| { + FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id())) + }) + .add_counter(function_source_hash, index, start_byte_pos, end_byte_pos); } fn add_counter_expression_region( @@ -96,18 +75,16 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { end_byte_pos: u32, ) { debug!( - "adding counter expression to coverage map: instance={:?}, index={}, {} {:?} {}, byte range {}..{}", + "adding counter expression to coverage_regions: instance={:?}, index={}, {} {:?} {}, byte range {}..{}", instance, index, lhs, op, rhs, start_byte_pos, end_byte_pos, ); - let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut(); - coverage_regions.entry(instance).or_default().add_counter_expression( - index, - lhs, - op, - rhs, - start_byte_pos, - end_byte_pos, - ); + let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| { + FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id())) + }) + .add_counter_expression(index, lhs, op, rhs, start_byte_pos, end_byte_pos); } fn add_unreachable_region( @@ -117,10 +94,175 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { end_byte_pos: u32, ) { debug!( - "adding unreachable code to coverage map: instance={:?}, byte range {}..{}", + "adding unreachable code to coverage_regions: instance={:?}, byte range {}..{}", instance, start_byte_pos, end_byte_pos, ); - let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut(); - coverage_regions.entry(instance).or_default().add_unreachable(start_byte_pos, end_byte_pos); + let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut(); + coverage_regions + .entry(instance) + .or_insert_with(|| { + FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id())) + }) + .add_unreachable(start_byte_pos, end_byte_pos); + } +} + +/// This struct wraps an opaque reference to the C++ template instantiation of +/// `llvm::SmallVector<coverage::CounterExpression>`. Each `coverage::CounterExpression` object is +/// constructed from primative-typed arguments, and pushed to the `SmallVector`, in the C++ +/// implementation of `LLVMRustCoverageSmallVectorCounterExpressionAdd()` (see +/// `src/rustllvm/CoverageMappingWrapper.cpp`). +pub struct SmallVectorCounterExpression<'a> { + pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterExpression<'a>, +} + +impl SmallVectorCounterExpression<'a> { + pub fn new() -> Self { + SmallVectorCounterExpression { + raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterExpressionCreate() }, + } + } + + pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterExpression<'a> { + self.raw + } + + pub fn push_from( + &mut self, + kind: rustc_codegen_ssa::coverageinfo::CounterOp, + left_index: u32, + right_index: u32, + ) { + unsafe { + llvm::LLVMRustCoverageSmallVectorCounterExpressionAdd( + &mut *(self.raw as *mut _), + kind, + left_index, + right_index, + ) + } + } +} + +impl Drop for SmallVectorCounterExpression<'a> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustCoverageSmallVectorCounterExpressionDispose(&mut *(self.raw as *mut _)); + } + } +} + +/// This struct wraps an opaque reference to the C++ template instantiation of +/// `llvm::SmallVector<coverage::CounterMappingRegion>`. Each `coverage::CounterMappingRegion` +/// object is constructed from primative-typed arguments, and pushed to the `SmallVector`, in the +/// C++ implementation of `LLVMRustCoverageSmallVectorCounterMappingRegionAdd()` (see +/// `src/rustllvm/CoverageMappingWrapper.cpp`). +pub struct SmallVectorCounterMappingRegion<'a> { + pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterMappingRegion<'a>, +} + +impl SmallVectorCounterMappingRegion<'a> { + pub fn new() -> Self { + SmallVectorCounterMappingRegion { + raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterMappingRegionCreate() }, + } + } + + pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterMappingRegion<'a> { + self.raw + } + + pub fn push_from( + &mut self, + index: u32, + file_id: u32, + line_start: u32, + column_start: u32, + line_end: u32, + column_end: u32, + ) { + unsafe { + llvm::LLVMRustCoverageSmallVectorCounterMappingRegionAdd( + &mut *(self.raw as *mut _), + index, + file_id, + line_start, + column_start, + line_end, + column_end, + ) + } + } +} + +impl Drop for SmallVectorCounterMappingRegion<'a> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustCoverageSmallVectorCounterMappingRegionDispose( + &mut *(self.raw as *mut _), + ); + } + } +} + +pub(crate) fn write_filenames_section_to_buffer(filenames: &Vec<CString>, buffer: &RustString) { + let c_str_vec = filenames.iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>(); + unsafe { + llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( + c_str_vec.as_ptr(), + c_str_vec.len(), + buffer, + ); + } +} + +pub(crate) fn write_mapping_to_buffer( + virtual_file_mapping: Vec<u32>, + expressions: SmallVectorCounterExpression<'_>, + mapping_regions: SmallVectorCounterMappingRegion<'_>, + buffer: &RustString, +) { + unsafe { + llvm::LLVMRustCoverageWriteMappingToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len() as c_uint, + expressions.as_ptr(), + mapping_regions.as_ptr(), + buffer, + ); } } + +pub(crate) fn compute_hash(name: &str) -> u64 { + let name = CString::new(name).expect("null error converting hashable name to C string"); + unsafe { llvm::LLVMRustCoverageComputeHash(name.as_ptr()) } +} + +pub(crate) fn mapping_version() -> u32 { + unsafe { llvm::LLVMRustCoverageMappingVersion() } +} + +pub(crate) fn save_map_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + cov_data_val: &'ll llvm::Value, +) { + let covmap_var_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteMappingVarNameToString(s); + }) + .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + debug!("covmap var name: {:?}", covmap_var_name); + + let covmap_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage section name failed UTF-8 conversion"); + debug!("covmap section name: {:?}", covmap_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); + llvm::set_initializer(llglobal, cov_data_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::InternalLinkage); + llvm::set_section(llglobal, &covmap_section_name); + llvm::set_alignment(llglobal, COVMAP_VAR_ALIGN_BYTES); + cx.add_used_global(llglobal); +} diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index d095587a1e8..63ec8031483 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -90,45 +90,64 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { args: &Vec<Operand<'tcx>>, caller_instance: ty::Instance<'tcx>, ) -> bool { - match intrinsic { - sym::count_code_region => { - use coverage::count_code_region_args::*; - self.add_counter_region( - caller_instance, - op_to_u32(&args[COUNTER_INDEX]), - op_to_u32(&args[START_BYTE_POS]), - op_to_u32(&args[END_BYTE_POS]), - ); - true // Also inject the counter increment in the backend - } - sym::coverage_counter_add | sym::coverage_counter_subtract => { - use coverage::coverage_counter_expression_args::*; - self.add_counter_expression_region( - caller_instance, - op_to_u32(&args[COUNTER_EXPRESSION_INDEX]), - op_to_u32(&args[LEFT_INDEX]), - if intrinsic == sym::coverage_counter_add { - CounterOp::Add - } else { - CounterOp::Subtract - }, - op_to_u32(&args[RIGHT_INDEX]), - op_to_u32(&args[START_BYTE_POS]), - op_to_u32(&args[END_BYTE_POS]), - ); - false // Does not inject backend code - } - sym::coverage_unreachable => { - use coverage::coverage_unreachable_args::*; - self.add_unreachable_region( - caller_instance, - op_to_u32(&args[START_BYTE_POS]), - op_to_u32(&args[END_BYTE_POS]), - ); - false // Does not inject backend code + if self.tcx.sess.opts.debugging_opts.instrument_coverage { + // Add the coverage information from the MIR to the Codegen context. Some coverage + // intrinsics are used only to pass along the coverage information (returns `false` + // for `is_codegen_intrinsic()`), but `count_code_region` is also converted into an + // LLVM intrinsic to increment a coverage counter. + match intrinsic { + sym::count_code_region => { + use coverage::count_code_region_args::*; + self.add_counter_region( + caller_instance, + op_to_u64(&args[FUNCTION_SOURCE_HASH]), + op_to_u32(&args[COUNTER_INDEX]), + op_to_u32(&args[START_BYTE_POS]), + op_to_u32(&args[END_BYTE_POS]), + ); + return true; // Also inject the counter increment in the backend + } + sym::coverage_counter_add | sym::coverage_counter_subtract => { + use coverage::coverage_counter_expression_args::*; + self.add_counter_expression_region( + caller_instance, + op_to_u32(&args[COUNTER_EXPRESSION_INDEX]), + op_to_u32(&args[LEFT_INDEX]), + if intrinsic == sym::coverage_counter_add { + CounterOp::Add + } else { + CounterOp::Subtract + }, + op_to_u32(&args[RIGHT_INDEX]), + op_to_u32(&args[START_BYTE_POS]), + op_to_u32(&args[END_BYTE_POS]), + ); + return false; // Does not inject backend code + } + sym::coverage_unreachable => { + use coverage::coverage_unreachable_args::*; + self.add_unreachable_region( + caller_instance, + op_to_u32(&args[START_BYTE_POS]), + op_to_u32(&args[END_BYTE_POS]), + ); + return false; // Does not inject backend code + } + _ => {} + } + } else { + // NOT self.tcx.sess.opts.debugging_opts.instrument_coverage + if intrinsic == sym::count_code_region { + // An external crate may have been pre-compiled with coverage instrumentation, and + // some references from the current crate to the external crate might carry along + // the call terminators to coverage intrinsics, like `count_code_region` (for + // example, when instantiating a generic function). If the current crate has + // `instrument_coverage` disabled, the `count_code_region` call terminators should + // be ignored. + return false; // Do not inject coverage counters inlined from external crates } - _ => true, // Unhandled intrinsics should be passed to `codegen_intrinsic_call()` } + true // Unhandled intrinsics should be passed to `codegen_intrinsic_call()` } fn codegen_intrinsic_call( @@ -197,12 +216,13 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let coverageinfo = tcx.coverageinfo(caller_instance.def_id()); let mangled_fn = tcx.symbol_name(caller_instance); let (mangled_fn_name, _len_val) = self.const_str(Symbol::intern(mangled_fn.name)); - let hash = self.const_u64(coverageinfo.hash); let num_counters = self.const_u32(coverageinfo.num_counters); use coverage::count_code_region_args::*; + let hash = args[FUNCTION_SOURCE_HASH].immediate(); let index = args[COUNTER_INDEX].immediate(); debug!( - "count_code_region to LLVM intrinsic instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})", + "translating Rust intrinsic `count_code_region()` to LLVM intrinsic: \ + instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})", mangled_fn.name, hash, num_counters, index, ); self.instrprof_increment(mangled_fn_name, hash, num_counters, index) @@ -2222,3 +2242,7 @@ fn float_type_width(ty: Ty<'_>) -> Option<u64> { fn op_to_u32<'tcx>(op: &Operand<'tcx>) -> u32 { Operand::scalar_from_const(op).to_u32().expect("Scalar is u32") } + +fn op_to_u64<'tcx>(op: &Operand<'tcx>) -> u64 { + Operand::scalar_from_const(op).to_u64().expect("Scalar is u64") +} diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs index 64f5e103f0b..9784beaa079 100644 --- a/src/librustc_codegen_llvm/llvm/ffi.rs +++ b/src/librustc_codegen_llvm/llvm/ffi.rs @@ -1,6 +1,8 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] +use super::coverageinfo::{SmallVectorCounterExpression, SmallVectorCounterMappingRegion}; + use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DINameSpace, DISPFlags, DIScope, @@ -650,6 +652,16 @@ pub struct Linker<'a>(InvariantOpaque<'a>); pub type DiagnosticHandler = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); pub type InlineAsmDiagHandler = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); +pub mod coverageinfo { + use super::InvariantOpaque; + + #[repr(C)] + pub struct SmallVectorCounterExpression<'a>(InvariantOpaque<'a>); + + #[repr(C)] + pub struct SmallVectorCounterMappingRegion<'a>(InvariantOpaque<'a>); +} + pub mod debuginfo { use super::{InvariantOpaque, Metadata}; use bitflags::bitflags; @@ -1365,7 +1377,7 @@ extern "C" { // Miscellaneous instructions pub fn LLVMBuildPhi(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; - pub fn LLVMRustGetInstrprofIncrementIntrinsic(M: &Module) -> &'a Value; + pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &'a Value; pub fn LLVMRustBuildCall( B: &Builder<'a>, Fn: &'a Value, @@ -1633,6 +1645,58 @@ extern "C" { ConstraintsLen: size_t, ) -> bool; + pub fn LLVMRustCoverageSmallVectorCounterExpressionCreate() + -> &'a mut SmallVectorCounterExpression<'a>; + pub fn LLVMRustCoverageSmallVectorCounterExpressionDispose( + Container: &'a mut SmallVectorCounterExpression<'a>, + ); + pub fn LLVMRustCoverageSmallVectorCounterExpressionAdd( + Container: &mut SmallVectorCounterExpression<'a>, + Kind: rustc_codegen_ssa::coverageinfo::CounterOp, + LeftIndex: c_uint, + RightIndex: c_uint, + ); + + pub fn LLVMRustCoverageSmallVectorCounterMappingRegionCreate() + -> &'a mut SmallVectorCounterMappingRegion<'a>; + pub fn LLVMRustCoverageSmallVectorCounterMappingRegionDispose( + Container: &'a mut SmallVectorCounterMappingRegion<'a>, + ); + pub fn LLVMRustCoverageSmallVectorCounterMappingRegionAdd( + Container: &mut SmallVectorCounterMappingRegion<'a>, + Index: c_uint, + FileID: c_uint, + LineStart: c_uint, + ColumnStart: c_uint, + LineEnd: c_uint, + ColumnEnd: c_uint, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + Filenames: *const *const c_char, + FilenamesLen: size_t, + BufferOut: &RustString, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingToBuffer( + VirtualFileMappingIDs: *const c_uint, + NumVirtualFileMappingIDs: c_uint, + Expressions: *const SmallVectorCounterExpression<'_>, + MappingRegions: *const SmallVectorCounterMappingRegion<'_>, + BufferOut: &RustString, + ); + + pub fn LLVMRustCoverageComputeHash(Name: *const c_char) -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + + pub fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; pub fn LLVMRustVersionMajor() -> u32; pub fn LLVMRustVersionMinor() -> u32; diff --git a/src/librustc_codegen_llvm/llvm/mod.rs b/src/librustc_codegen_llvm/llvm/mod.rs index b7f1e1789c9..c09e3659f80 100644 --- a/src/librustc_codegen_llvm/llvm/mod.rs +++ b/src/librustc_codegen_llvm/llvm/mod.rs @@ -12,7 +12,7 @@ use libc::c_uint; use rustc_data_structures::small_c_str::SmallCStr; use rustc_llvm::RustString; use std::cell::RefCell; -use std::ffi::CStr; +use std::ffi::{CStr, CString}; use std::str::FromStr; use std::string::FromUtf8Error; @@ -189,6 +189,42 @@ pub fn mk_section_iter(llof: &ffi::ObjectFile) -> SectionIter<'_> { unsafe { SectionIter { llsi: LLVMGetSections(llof) } } } +pub fn set_section(llglobal: &Value, section_name: &str) { + let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); + unsafe { + LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + } +} + +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { + let name_cstr = CString::new(name).expect("unexpected CString error"); + unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } +} + +pub fn set_initializer(llglobal: &Value, constant_val: &Value) { + unsafe { + LLVMSetInitializer(llglobal, constant_val); + } +} + +pub fn set_global_constant(llglobal: &Value, is_constant: bool) { + unsafe { + LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + } +} + +pub fn set_linkage(llglobal: &Value, linkage: Linkage) { + unsafe { + LLVMRustSetLinkage(llglobal, linkage); + } +} + +pub fn set_alignment(llglobal: &Value, bytes: usize) { + unsafe { + ffi::LLVMSetAlignment(llglobal, bytes as c_uint); + } +} + /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. pub fn get_param(llfn: &Value, index: c_uint) -> &Value { unsafe { @@ -225,6 +261,12 @@ pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error String::from_utf8(sr.bytes.into_inner()) } +pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> { + let sr = RustString { bytes: RefCell::new(Vec::new()) }; + f(&sr); + sr.bytes.into_inner() +} + pub fn twine_to_string(tr: &Twine) -> String { unsafe { build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") |