Auto merge of #74733 - richkadel:llvm-coverage-map-gen-5, r=tmandry

Fixed coverage map issues; better aligned with LLVM APIs

Found some problems with the coverage map encoding when testing with more than one counter per function.

While debugging, I realized some better ways to structure the Rust implementation of the coverage mapping generator. I refactored somewhat, resulting in less code overall, expanded coverage of LLVM Coverage Map capabilities, and much closer alignment with LLVM data structures, APIs, and naming.

This should be easier to follow and easier to maintain.

r? @tmandry

Rust compiler MCP rust-lang/compiler-team#278
Relevant issue: #34701 - Implement support for LLVMs code coverage instrumentation

4 files changed, 303 insertions, 319 deletions

diff --git a/src/librustc_codegen_llvm/coverageinfo/mapgen.rs b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs
index 7f48b1d864c..4d9747a43f2 100644
--- a/src/librustc_codegen_llvm/coverageinfo/mapgen.rs
+++ b/src/librustc_codegen_llvm/coverageinfo/mapgen.rs
@@ -1,23 +1,15 @@
-use crate::llvm;
-
 use crate::common::CodegenCx;
 use crate::coverageinfo;
+use crate::llvm;
 
+use llvm::coverageinfo::CounterMappingRegion;
 use log::debug;
-use rustc_codegen_ssa::coverageinfo::map::*;
-use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, MiscMethods};
+use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, Region};
+use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_llvm::RustString;
-use rustc_middle::ty::Instance;
-use rustc_middle::{bug, mir};
 
-use std::collections::BTreeMap;
 use std::ffi::CString;
-use std::path::PathBuf;
-
-// FIXME(richkadel): Complete all variations of generating and exporting the coverage map to LLVM.
-// The current implementation is an initial foundation with basic capabilities (Counters, but not
-// CounterExpressions, etc.).
 
 /// Generates and exports the Coverage Map.
 ///
@@ -32,174 +24,123 @@ use std::path::PathBuf;
 /// undocumented details in Clang's implementation (that may or may not be important) were also
 /// replicated for Rust's Coverage Map.
 pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
-    let mut coverage_writer = CoverageMappingWriter::new(cx);
-
     let function_coverage_map = cx.coverage_context().take_function_coverage_map();
+    if function_coverage_map.is_empty() {
+        // This module has no functions with coverage instrumentation
+        return;
+    }
+
+    let mut mapgen = CoverageMapGenerator::new();
 
     // Encode coverage mappings and generate function records
     let mut function_records = Vec::<&'ll llvm::Value>::new();
     let coverage_mappings_buffer = llvm::build_byte_buffer(|coverage_mappings_buffer| {
         for (instance, function_coverage) in function_coverage_map.into_iter() {
-            if let Some(function_record) = coverage_writer.write_function_mappings_and_record(
-                instance,
-                function_coverage,
-                coverage_mappings_buffer,
-            ) {
-                function_records.push(function_record);
-            }
+            debug!("Generate coverage map for: {:?}", instance);
+
+            let mangled_function_name = cx.tcx.symbol_name(instance).to_string();
+            let function_source_hash = function_coverage.source_hash();
+            let (expressions, counter_regions) =
+                function_coverage.get_expressions_and_counter_regions();
+
+            let old_len = coverage_mappings_buffer.len();
+            mapgen.write_coverage_mappings(expressions, counter_regions, coverage_mappings_buffer);
+            let mapping_data_size = coverage_mappings_buffer.len() - old_len;
+            debug_assert!(
+                mapping_data_size > 0,
+                "Every `FunctionCoverage` should have at least one counter"
+            );
+
+            let function_record = mapgen.make_function_record(
+                cx,
+                mangled_function_name,
+                function_source_hash,
+                mapping_data_size,
+            );
+            function_records.push(function_record);
         }
     });
 
-    // Encode all filenames covered in this module, ordered by `file_id`
+    // Encode all filenames referenced by counters/expressions in this module
     let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| {
-        coverageinfo::write_filenames_section_to_buffer(
-            &coverage_writer.filenames,
-            filenames_buffer,
-        );
+        coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer);
     });
 
-    if coverage_mappings_buffer.len() > 0 {
-        // Generate the LLVM IR representation of the coverage map and store it in a well-known
-        // global constant.
-        coverage_writer.write_coverage_map(
-            function_records,
-            filenames_buffer,
-            coverage_mappings_buffer,
-        );
-    }
+    // Generate the LLVM IR representation of the coverage map and store it in a well-known global
+    mapgen.save_generated_coverage_map(
+        cx,
+        function_records,
+        filenames_buffer,
+        coverage_mappings_buffer,
+    );
 }
 
-struct CoverageMappingWriter<'a, 'll, 'tcx> {
-    cx: &'a CodegenCx<'ll, 'tcx>,
+struct CoverageMapGenerator {
     filenames: Vec<CString>,
     filename_to_index: FxHashMap<CString, u32>,
 }
 
-impl<'a, 'll, 'tcx> CoverageMappingWriter<'a, 'll, 'tcx> {
-    fn new(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
-        Self { cx, filenames: Vec::new(), filename_to_index: FxHashMap::<CString, u32>::default() }
+impl CoverageMapGenerator {
+    fn new() -> Self {
+        Self { filenames: Vec::new(), filename_to_index: FxHashMap::default() }
     }
 
-    /// For the given function, get the coverage region data, stream it to the given buffer, and
-    /// then generate and return a new function record.
-    fn write_function_mappings_and_record(
+    /// Using the `expressions` and `counter_regions` collected for the current function, generate
+    /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use
+    /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into
+    /// the given `coverage_mappings` byte buffer, compliant with the LLVM Coverage Mapping format.
+    fn write_coverage_mappings(
         &mut self,
-        instance: Instance<'tcx>,
-        mut function_coverage: FunctionCoverage,
+        expressions: Vec<CounterExpression>,
+        counter_regions: impl Iterator<Item = (Counter, &'a Region)>,
         coverage_mappings_buffer: &RustString,
-    ) -> Option<&'ll llvm::Value> {
-        let cx = self.cx;
-        let coverageinfo: &mir::CoverageInfo = cx.tcx.coverageinfo(instance.def_id());
-        debug!(
-            "Generate coverage map for: {:?}, num_counters: {}, num_expressions: {}",
-            instance, coverageinfo.num_counters, coverageinfo.num_expressions
-        );
-        debug_assert!(coverageinfo.num_counters > 0);
-
-        let regions_in_file_order = function_coverage.regions_in_file_order(cx.sess().source_map());
-        if regions_in_file_order.len() == 0 {
-            return None;
+    ) {
+        let mut counter_regions = counter_regions.collect::<Vec<_>>();
+        if counter_regions.is_empty() {
+            return;
         }
 
-        // Stream the coverage mapping regions for the function (`instance`) to the buffer, and
-        // compute the data byte size used.
-        let old_len = coverage_mappings_buffer.len();
-        self.regions_to_mappings(regions_in_file_order, coverage_mappings_buffer);
-        let mapping_data_size = coverage_mappings_buffer.len() - old_len;
-        debug_assert!(mapping_data_size > 0);
-
-        let mangled_function_name = cx.tcx.symbol_name(instance).to_string();
-        let name_ref = coverageinfo::compute_hash(&mangled_function_name);
-        let function_source_hash = function_coverage.source_hash();
-
-        // Generate and return the function record
-        let name_ref_val = cx.const_u64(name_ref);
-        let mapping_data_size_val = cx.const_u32(mapping_data_size as u32);
-        let func_hash_val = cx.const_u64(function_source_hash);
-        Some(cx.const_struct(
-            &[name_ref_val, mapping_data_size_val, func_hash_val],
-            /*packed=*/ true,
-        ))
-    }
-
-    /// For each coverage region, extract its coverage data from the earlier coverage analysis.
-    /// Use LLVM APIs to convert the data into buffered bytes compliant with the LLVM Coverage
-    /// Mapping format.
-    fn regions_to_mappings(
-        &mut self,
-        regions_in_file_order: BTreeMap<PathBuf, BTreeMap<CoverageLoc, (usize, CoverageKind)>>,
-        coverage_mappings_buffer: &RustString,
-    ) {
         let mut virtual_file_mapping = Vec::new();
-        let mut mapping_regions = coverageinfo::SmallVectorCounterMappingRegion::new();
-        let mut expressions = coverageinfo::SmallVectorCounterExpression::new();
-
-        for (file_id, (file_path, file_coverage_regions)) in
-            regions_in_file_order.into_iter().enumerate()
-        {
-            let file_id = file_id as u32;
-            let filename = CString::new(file_path.to_string_lossy().to_string())
-                .expect("null error converting filename to C string");
-            debug!("  file_id: {} = '{:?}'", file_id, filename);
-            let filenames_index = match self.filename_to_index.get(&filename) {
-                Some(index) => *index,
-                None => {
-                    let index = self.filenames.len() as u32;
-                    self.filenames.push(filename.clone());
-                    self.filename_to_index.insert(filename, index);
-                    index
+        let mut mapping_regions = Vec::new();
+        let mut current_file_path = None;
+        let mut current_file_id = 0;
+
+        // Convert the list of (Counter, Region) pairs to an array of `CounterMappingRegion`, sorted
+        // by filename and position. Capture any new files to compute the `CounterMappingRegion`s
+        // `file_id` (indexing files referenced by the current function), and construct the
+        // function-specific `virtual_file_mapping` from `file_id` to its index in the module's
+        // `filenames` array.
+        counter_regions.sort_unstable_by_key(|(_counter, region)| *region);
+        for (counter, region) in counter_regions {
+            let (file_path, start_line, start_col, end_line, end_col) = region.file_start_and_end();
+            let same_file = current_file_path.as_ref().map_or(false, |p| p == file_path);
+            if !same_file {
+                if current_file_path.is_some() {
+                    current_file_id += 1;
                 }
-            };
-            virtual_file_mapping.push(filenames_index);
-
-            let mut mapping_indexes = vec![0 as u32; file_coverage_regions.len()];
-            for (mapping_index, (region_id, _)) in file_coverage_regions.values().enumerate() {
-                mapping_indexes[*region_id] = mapping_index as u32;
-            }
-
-            for (region_loc, (region_id, region_kind)) in file_coverage_regions.into_iter() {
-                let mapping_index = mapping_indexes[region_id];
-                match region_kind {
-                    CoverageKind::Counter => {
-                        debug!(
-                            "  Counter {}, file_id: {}, region_loc: {}",
-                            mapping_index, file_id, region_loc
-                        );
-                        mapping_regions.push_from(
-                            mapping_index,
-                            file_id,
-                            region_loc.start_line,
-                            region_loc.start_col,
-                            region_loc.end_line,
-                            region_loc.end_col,
-                        );
-                    }
-                    CoverageKind::CounterExpression(lhs, op, rhs) => {
-                        debug!(
-                            "  CounterExpression {} = {} {:?} {}, file_id: {}, region_loc: {:?}",
-                            mapping_index, lhs, op, rhs, file_id, region_loc,
-                        );
-                        mapping_regions.push_from(
-                            mapping_index,
-                            file_id,
-                            region_loc.start_line,
-                            region_loc.start_col,
-                            region_loc.end_line,
-                            region_loc.end_col,
-                        );
-                        expressions.push_from(op, lhs, rhs);
-                    }
-                    CoverageKind::Unreachable => {
-                        debug!(
-                            "  Unreachable region, file_id: {}, region_loc: {:?}",
-                            file_id, region_loc,
-                        );
-                        bug!("Unreachable region not expected and not yet handled!")
-                        // FIXME(richkadel): implement and call
-                        //   mapping_regions.push_from(...) for unreachable regions
+                current_file_path = Some(file_path.clone());
+                let filename = CString::new(file_path.to_string_lossy().to_string())
+                    .expect("null error converting filename to C string");
+                debug!("  file_id: {} = '{:?}'", current_file_id, filename);
+                let filenames_index = match self.filename_to_index.get(&filename) {
+                    Some(index) => *index,
+                    None => {
+                        let index = self.filenames.len() as u32;
+                        self.filenames.push(filename.clone());
+                        self.filename_to_index.insert(filename.clone(), index);
+                        index
                     }
-                }
+                };
+                virtual_file_mapping.push(filenames_index);
             }
+            mapping_regions.push(CounterMappingRegion::code_region(
+                counter,
+                current_file_id,
+                start_line,
+                start_col,
+                end_line,
+                end_col,
+            ));
         }
 
         // Encode and append the current function's coverage mapping data
@@ -211,14 +152,35 @@ impl<'a, 'll, 'tcx> CoverageMappingWriter<'a, 'll, 'tcx> {
         );
     }
 
-    fn write_coverage_map(
+    /// Generate and return the function record `Value`
+    fn make_function_record(
+        &mut self,
+        cx: &CodegenCx<'ll, 'tcx>,
+        mangled_function_name: String,
+        function_source_hash: u64,
+        mapping_data_size: usize,
+    ) -> &'ll llvm::Value {
+        let name_ref = coverageinfo::compute_hash(&mangled_function_name);
+        let name_ref_val = cx.const_u64(name_ref);
+        let mapping_data_size_val = cx.const_u32(mapping_data_size as u32);
+        let func_hash_val = cx.const_u64(function_source_hash);
+        cx.const_struct(
+            &[name_ref_val, mapping_data_size_val, func_hash_val],
+            /*packed=*/ true,
+        )
+    }
+
+    /// Combine the filenames and coverage mappings buffers, construct coverage map header and the
+    /// array of function records, and combine everything into the complete coverage map. Save the
+    /// coverage map data into the LLVM IR as a static global using a specific, well-known section
+    /// and name.
+    fn save_generated_coverage_map(
         self,
+        cx: &CodegenCx<'ll, 'tcx>,
         function_records: Vec<&'ll llvm::Value>,
         filenames_buffer: Vec<u8>,
         mut coverage_mappings_buffer: Vec<u8>,
     ) {
-        let cx = self.cx;
-
         // Concatenate the encoded filenames and encoded coverage mappings, and add additional zero
         // bytes as-needed to ensure 8-byte alignment.
         let mut coverage_size = coverage_mappings_buffer.len();
diff --git a/src/librustc_codegen_llvm/coverageinfo/mod.rs b/src/librustc_codegen_llvm/coverageinfo/mod.rs
index 76894bcd6c1..9d2090eae8f 100644
--- a/src/librustc_codegen_llvm/coverageinfo/mod.rs
+++ b/src/librustc_codegen_llvm/coverageinfo/mod.rs
@@ -4,8 +4,9 @@ use crate::builder::Builder;
 use crate::common::CodegenCx;
 
 use libc::c_uint;
+use llvm::coverageinfo::CounterMappingRegion;
 use log::debug;
-use rustc_codegen_ssa::coverageinfo::map::*;
+use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, ExprKind, FunctionCoverage};
 use rustc_codegen_ssa::traits::{
     BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, StaticMethods,
 };
@@ -23,7 +24,7 @@ const COVMAP_VAR_ALIGN_BYTES: usize = 8;
 /// A context object for maintaining all state needed by the coverageinfo module.
 pub struct CrateCoverageContext<'tcx> {
     // Coverage region data for each instrumented function identified by DefId.
-    pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage>>,
+    pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
 }
 
 impl<'tcx> CrateCoverageContext<'tcx> {
@@ -31,7 +32,7 @@ impl<'tcx> CrateCoverageContext<'tcx> {
         Self { function_coverage_map: Default::default() }
     }
 
-    pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage> {
+    pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
         self.function_coverage_map.replace(FxHashMap::default())
     }
 }
@@ -47,44 +48,49 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         &mut self,
         instance: Instance<'tcx>,
         function_source_hash: u64,
-        index: u32,
+        id: u32,
         start_byte_pos: u32,
         end_byte_pos: u32,
     ) {
         debug!(
-            "adding counter to coverage_regions: instance={:?}, function_source_hash={}, index={}, byte range {}..{}",
-            instance, function_source_hash, index, start_byte_pos, end_byte_pos,
+            "adding counter to coverage_regions: instance={:?}, function_source_hash={}, id={}, \
+             byte range {}..{}",
+            instance, function_source_hash, id, start_byte_pos, end_byte_pos,
         );
         let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
         coverage_regions
             .entry(instance)
-            .or_insert_with(|| {
-                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
-            })
-            .add_counter(function_source_hash, index, start_byte_pos, end_byte_pos);
+            .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
+            .add_counter(function_source_hash, id, start_byte_pos, end_byte_pos);
     }
 
     fn add_counter_expression_region(
         &mut self,
         instance: Instance<'tcx>,
-        index: u32,
+        id_descending_from_max: u32,
         lhs: u32,
-        op: CounterOp,
+        op: ExprKind,
         rhs: u32,
         start_byte_pos: u32,
         end_byte_pos: u32,
     ) {
         debug!(
-            "adding counter expression to coverage_regions: instance={:?}, index={}, {} {:?} {}, byte range {}..{}",
-            instance, index, lhs, op, rhs, start_byte_pos, end_byte_pos,
+            "adding counter expression to coverage_regions: instance={:?}, id={}, {} {:?} {}, \
+             byte range {}..{}",
+            instance, id_descending_from_max, lhs, op, rhs, start_byte_pos, end_byte_pos,
         );
         let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
         coverage_regions
             .entry(instance)
-            .or_insert_with(|| {
-                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
-            })
-            .add_counter_expression(index, lhs, op, rhs, start_byte_pos, end_byte_pos);
+            .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
+            .add_counter_expression(
+                id_descending_from_max,
+                lhs,
+                op,
+                rhs,
+                start_byte_pos,
+                end_byte_pos,
+            );
     }
 
     fn add_unreachable_region(
@@ -100,108 +106,8 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
         coverage_regions
             .entry(instance)
-            .or_insert_with(|| {
-                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
-            })
-            .add_unreachable(start_byte_pos, end_byte_pos);
-    }
-}
-
-/// This struct wraps an opaque reference to the C++ template instantiation of
-/// `llvm::SmallVector<coverage::CounterExpression>`. Each `coverage::CounterExpression` object is
-/// constructed from primative-typed arguments, and pushed to the `SmallVector`, in the C++
-/// implementation of `LLVMRustCoverageSmallVectorCounterExpressionAdd()` (see
-/// `src/rustllvm/CoverageMappingWrapper.cpp`).
-pub struct SmallVectorCounterExpression<'a> {
-    pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterExpression<'a>,
-}
-
-impl SmallVectorCounterExpression<'a> {
-    pub fn new() -> Self {
-        SmallVectorCounterExpression {
-            raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterExpressionCreate() },
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterExpression<'a> {
-        self.raw
-    }
-
-    pub fn push_from(
-        &mut self,
-        kind: rustc_codegen_ssa::coverageinfo::CounterOp,
-        left_index: u32,
-        right_index: u32,
-    ) {
-        unsafe {
-            llvm::LLVMRustCoverageSmallVectorCounterExpressionAdd(
-                &mut *(self.raw as *mut _),
-                kind,
-                left_index,
-                right_index,
-            )
-        }
-    }
-}
-
-impl Drop for SmallVectorCounterExpression<'a> {
-    fn drop(&mut self) {
-        unsafe {
-            llvm::LLVMRustCoverageSmallVectorCounterExpressionDispose(&mut *(self.raw as *mut _));
-        }
-    }
-}
-
-/// This struct wraps an opaque reference to the C++ template instantiation of
-/// `llvm::SmallVector<coverage::CounterMappingRegion>`. Each `coverage::CounterMappingRegion`
-/// object is constructed from primative-typed arguments, and pushed to the `SmallVector`, in the
-/// C++ implementation of `LLVMRustCoverageSmallVectorCounterMappingRegionAdd()` (see
-/// `src/rustllvm/CoverageMappingWrapper.cpp`).
-pub struct SmallVectorCounterMappingRegion<'a> {
-    pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterMappingRegion<'a>,
-}
-
-impl SmallVectorCounterMappingRegion<'a> {
-    pub fn new() -> Self {
-        SmallVectorCounterMappingRegion {
-            raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterMappingRegionCreate() },
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterMappingRegion<'a> {
-        self.raw
-    }
-
-    pub fn push_from(
-        &mut self,
-        index: u32,
-        file_id: u32,
-        line_start: u32,
-        column_start: u32,
-        line_end: u32,
-        column_end: u32,
-    ) {
-        unsafe {
-            llvm::LLVMRustCoverageSmallVectorCounterMappingRegionAdd(
-                &mut *(self.raw as *mut _),
-                index,
-                file_id,
-                line_start,
-                column_start,
-                line_end,
-                column_end,
-            )
-        }
-    }
-}
-
-impl Drop for SmallVectorCounterMappingRegion<'a> {
-    fn drop(&mut self) {
-        unsafe {
-            llvm::LLVMRustCoverageSmallVectorCounterMappingRegionDispose(
-                &mut *(self.raw as *mut _),
-            );
-        }
+            .or_insert_with(|| FunctionCoverage::new(self.tcx, instance))
+            .add_unreachable_region(start_byte_pos, end_byte_pos);
     }
 }
 
@@ -218,8 +124,8 @@ pub(crate) fn write_filenames_section_to_buffer(filenames: &Vec<CString>, buffer
 
 pub(crate) fn write_mapping_to_buffer(
     virtual_file_mapping: Vec<u32>,
-    expressions: SmallVectorCounterExpression<'_>,
-    mapping_regions: SmallVectorCounterMappingRegion<'_>,
+    expressions: Vec<CounterExpression>,
+    mut mapping_regions: Vec<CounterMappingRegion>,
     buffer: &RustString,
 ) {
     unsafe {
@@ -227,7 +133,9 @@ pub(crate) fn write_mapping_to_buffer(
             virtual_file_mapping.as_ptr(),
             virtual_file_mapping.len() as c_uint,
             expressions.as_ptr(),
-            mapping_regions.as_ptr(),
+            expressions.len() as c_uint,
+            mapping_regions.as_mut_ptr(),
+            mapping_regions.len() as c_uint,
             buffer,
         );
     }
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs
index 11b1c95c58b..236f7f69653 100644
--- a/src/librustc_codegen_llvm/intrinsic.rs
+++ b/src/librustc_codegen_llvm/intrinsic.rs
@@ -13,7 +13,7 @@ use rustc_ast::ast;
 use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh};
 use rustc_codegen_ssa::common::span_invalid_monomorphization_error;
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
-use rustc_codegen_ssa::coverageinfo::CounterOp;
+use rustc_codegen_ssa::coverageinfo::ExprKind;
 use rustc_codegen_ssa::glue;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
@@ -101,7 +101,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     self.add_counter_region(
                         caller_instance,
                         op_to_u64(&args[FUNCTION_SOURCE_HASH]),
-                        op_to_u32(&args[COUNTER_INDEX]),
+                        op_to_u32(&args[COUNTER_ID]),
                         op_to_u32(&args[START_BYTE_POS]),
                         op_to_u32(&args[END_BYTE_POS]),
                     );
@@ -111,14 +111,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     use coverage::coverage_counter_expression_args::*;
                     self.add_counter_expression_region(
                         caller_instance,
-                        op_to_u32(&args[COUNTER_EXPRESSION_INDEX]),
-                        op_to_u32(&args[LEFT_INDEX]),
+                        op_to_u32(&args[EXPRESSION_ID]),
+                        op_to_u32(&args[LEFT_ID]),
                         if intrinsic == sym::coverage_counter_add {
-                            CounterOp::Add
+                            ExprKind::Add
                         } else {
-                            CounterOp::Subtract
+                            ExprKind::Subtract
                         },
-                        op_to_u32(&args[RIGHT_INDEX]),
+                        op_to_u32(&args[RIGHT_ID]),
                         op_to_u32(&args[START_BYTE_POS]),
                         op_to_u32(&args[END_BYTE_POS]),
                     );
@@ -219,7 +219,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 let num_counters = self.const_u32(coverageinfo.num_counters);
                 use coverage::count_code_region_args::*;
                 let hash = args[FUNCTION_SOURCE_HASH].immediate();
-                let index = args[COUNTER_INDEX].immediate();
+                let index = args[COUNTER_ID].immediate();
                 debug!(
                     "translating Rust intrinsic `count_code_region()` to LLVM intrinsic: \
                     instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})",
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index 9784beaa079..eb7dc827f93 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -1,7 +1,7 @@
 #![allow(non_camel_case_types)]
 #![allow(non_upper_case_globals)]
 
-use super::coverageinfo::{SmallVectorCounterExpression, SmallVectorCounterMappingRegion};
+use rustc_codegen_ssa::coverageinfo::map as coverage_map;
 
 use super::debuginfo::{
     DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator,
@@ -653,13 +653,152 @@ pub type DiagnosticHandler = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void);
 pub type InlineAsmDiagHandler = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint);
 
 pub mod coverageinfo {
-    use super::InvariantOpaque;
+    use super::coverage_map;
 
+    /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/10.0-2020-05-05/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L205-L221)
+    #[derive(Copy, Clone, Debug)]
     #[repr(C)]
-    pub struct SmallVectorCounterExpression<'a>(InvariantOpaque<'a>);
+    pub enum RegionKind {
+        /// A CodeRegion associates some code with a counter
+        CodeRegion = 0,
+
+        /// An ExpansionRegion represents a file expansion region that associates
+        /// a source range with the expansion of a virtual source file, such as
+        /// for a macro instantiation or #include file.
+        ExpansionRegion = 1,
+
+        /// A SkippedRegion represents a source range with code that was skipped
+        /// by a preprocessor or similar means.
+        SkippedRegion = 2,
+
+        /// A GapRegion is like a CodeRegion, but its count is only set as the
+        /// line execution count when its the only region in the line.
+        GapRegion = 3,
+    }
 
+    /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the
+    /// coverage map, in accordance with the
+    /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/llvmorg-8.0.0/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format).
+    /// The struct composes fields representing the `Counter` type and value(s) (injected counter
+    /// ID, or expression type and operands), the source file (an indirect index into a "filenames
+    /// array", encoded separately), and source location (start and end positions of the represented
+    /// code region).
+    ///
+    /// Aligns with [llvm::coverage::CounterMappingRegion](https://github.com/rust-lang/llvm-project/blob/rustc/10.0-2020-05-05/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L223-L226)
+    /// Important: The Rust struct layout (order and types of fields) must match its C++
+    /// counterpart.
+    #[derive(Copy, Clone, Debug)]
     #[repr(C)]
-    pub struct SmallVectorCounterMappingRegion<'a>(InvariantOpaque<'a>);
+    pub struct CounterMappingRegion {
+        /// The counter type and type-dependent counter data, if any.
+        counter: coverage_map::Counter,
+
+        /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the
+        /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes
+        /// that, in turn, are used to look up the filename for this region.
+        file_id: u32,
+
+        /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find
+        /// the mapping regions created as a result of macro expansion, by checking if their file id
+        /// matches the expanded file id.
+        expanded_file_id: u32,
+
+        /// 1-based starting line of the mapping region.
+        start_line: u32,
+
+        /// 1-based starting column of the mapping region.
+        start_col: u32,
+
+        /// 1-based ending line of the mapping region.
+        end_line: u32,
+
+        /// 1-based ending column of the mapping region. If the high bit is set, the current
+        /// mapping region is a gap area.
+        end_col: u32,
+
+        kind: RegionKind,
+    }
+
+    impl CounterMappingRegion {
+        pub fn code_region(
+            counter: coverage_map::Counter,
+            file_id: u32,
+            start_line: u32,
+            start_col: u32,
+            end_line: u32,
+            end_col: u32,
+        ) -> Self {
+            Self {
+                counter,
+                file_id,
+                expanded_file_id: 0,
+                start_line,
+                start_col,
+                end_line,
+                end_col,
+                kind: RegionKind::CodeRegion,
+            }
+        }
+
+        pub fn expansion_region(
+            file_id: u32,
+            expanded_file_id: u32,
+            start_line: u32,
+            start_col: u32,
+            end_line: u32,
+            end_col: u32,
+        ) -> Self {
+            Self {
+                counter: coverage_map::Counter::zero(),
+                file_id,
+                expanded_file_id,
+                start_line,
+                start_col,
+                end_line,
+                end_col,
+                kind: RegionKind::ExpansionRegion,
+            }
+        }
+
+        pub fn skipped_region(
+            file_id: u32,
+            start_line: u32,
+            start_col: u32,
+            end_line: u32,
+            end_col: u32,
+        ) -> Self {
+            Self {
+                counter: coverage_map::Counter::zero(),
+                file_id,
+                expanded_file_id: 0,
+                start_line,
+                start_col,
+                end_line,
+                end_col,
+                kind: RegionKind::SkippedRegion,
+            }
+        }
+
+        pub fn gap_region(
+            counter: coverage_map::Counter,
+            file_id: u32,
+            start_line: u32,
+            start_col: u32,
+            end_line: u32,
+            end_col: u32,
+        ) -> Self {
+            Self {
+                counter,
+                file_id,
+                expanded_file_id: 0,
+                start_line,
+                start_col,
+                end_line,
+                end_col: ((1 as u32) << 31) | end_col,
+                kind: RegionKind::GapRegion,
+            }
+        }
+    }
 }
 
 pub mod debuginfo {
@@ -1645,33 +1784,6 @@ extern "C" {
         ConstraintsLen: size_t,
     ) -> bool;
 
-    pub fn LLVMRustCoverageSmallVectorCounterExpressionCreate()
-    -> &'a mut SmallVectorCounterExpression<'a>;
-    pub fn LLVMRustCoverageSmallVectorCounterExpressionDispose(
-        Container: &'a mut SmallVectorCounterExpression<'a>,
-    );
-    pub fn LLVMRustCoverageSmallVectorCounterExpressionAdd(
-        Container: &mut SmallVectorCounterExpression<'a>,
-        Kind: rustc_codegen_ssa::coverageinfo::CounterOp,
-        LeftIndex: c_uint,
-        RightIndex: c_uint,
-    );
-
-    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionCreate()
-    -> &'a mut SmallVectorCounterMappingRegion<'a>;
-    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionDispose(
-        Container: &'a mut SmallVectorCounterMappingRegion<'a>,
-    );
-    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionAdd(
-        Container: &mut SmallVectorCounterMappingRegion<'a>,
-        Index: c_uint,
-        FileID: c_uint,
-        LineStart: c_uint,
-        ColumnStart: c_uint,
-        LineEnd: c_uint,
-        ColumnEnd: c_uint,
-    );
-
     #[allow(improper_ctypes)]
     pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer(
         Filenames: *const *const c_char,
@@ -1683,8 +1795,10 @@ extern "C" {
     pub fn LLVMRustCoverageWriteMappingToBuffer(
         VirtualFileMappingIDs: *const c_uint,
         NumVirtualFileMappingIDs: c_uint,
-        Expressions: *const SmallVectorCounterExpression<'_>,
-        MappingRegions: *const SmallVectorCounterMappingRegion<'_>,
+        Expressions: *const coverage_map::CounterExpression,
+        NumExpressions: c_uint,
+        MappingRegions: *mut coverageinfo::CounterMappingRegion,
+        NumMappingRegions: c_uint,
         BufferOut: &RustString,
     );