diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/abi.rs | 146 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/asm.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 23 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 385 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs | 54 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 5 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 109 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/mod.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 10 |
11 files changed, 523 insertions, 242 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 7857ccb613b..915dd3d9eda 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -36,17 +36,17 @@ impl ArgAttributeExt for ArgAttribute { where F: FnMut(llvm::Attribute), { - for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, SExt, StructRet, ZExt, InReg) + for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg) } } pub trait ArgAttributesExt { - fn apply_llfn(&self, idx: AttributePlace, llfn: &Value, ty: Option<&Type>); - fn apply_callsite(&self, idx: AttributePlace, callsite: &Value, ty: Option<&Type>); + fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value); + fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value); } impl ArgAttributesExt for ArgAttributes { - fn apply_llfn(&self, idx: AttributePlace, llfn: &Value, ty: Option<&Type>) { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value) { let mut regular = self.regular; unsafe { let deref = self.pointee_size.bytes(); @@ -61,14 +61,20 @@ impl ArgAttributesExt for ArgAttributes { if let Some(align) = self.pointee_align { llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); } - if regular.contains(ArgAttribute::ByVal) { - llvm::LLVMRustAddByValAttr(llfn, idx.as_uint(), ty.unwrap()); - } regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); + match self.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => { + llvm::Attribute::ZExt.apply_llfn(idx, llfn); + } + ArgExtension::Sext => { + llvm::Attribute::SExt.apply_llfn(idx, llfn); + } + } } } - fn apply_callsite(&self, idx: AttributePlace, callsite: &Value, ty: Option<&Type>) { + fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value) { let mut regular = self.regular; unsafe { let deref = self.pointee_size.bytes(); @@ -91,10 +97,16 @@ impl ArgAttributesExt for ArgAttributes { align.bytes() as u32, ); } - if regular.contains(ArgAttribute::ByVal) { - llvm::LLVMRustAddByValCallSiteAttr(callsite, idx.as_uint(), ty.unwrap()); - } regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); + match self.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => { + llvm::Attribute::ZExt.apply_callsite(idx, callsite); + } + ArgExtension::Sext => { + llvm::Attribute::SExt.apply_callsite(idx, callsite); + } + } } } } @@ -146,7 +158,7 @@ impl LlvmType for CastTarget { .prefix .iter() .flat_map(|option_kind| { - option_kind.map(|kind| Reg { kind, size: self.prefix_chunk }.llvm_type(cx)) + option_kind.map(|kind| Reg { kind, size: self.prefix_chunk_size }.llvm_type(cx)) }) .chain((0..rest_count).map(|_| rest_ll_unit)) .collect(); @@ -267,10 +279,12 @@ impl ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { PassMode::Pair(..) => { OperandValue::Pair(next(), next()).store(bx, dst); } - PassMode::Indirect(_, Some(_)) => { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); } - PassMode::Direct(_) | PassMode::Indirect(_, None) | PassMode::Cast(_) => { + PassMode::Direct(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } + | PassMode::Cast(_) => { let next_arg = next(); self.store(bx, next_arg, dst); } @@ -315,14 +329,14 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } ).sum(); let mut llargument_tys = Vec::with_capacity( - if let PassMode::Indirect(..) = self.ret.mode { 1 } else { 0 } + args_capacity, + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, ); let llreturn_ty = match self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), PassMode::Cast(cast) => cast.llvm_type(cx), - PassMode::Indirect(..) => { + PassMode::Indirect { .. } => { llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() } @@ -342,7 +356,7 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect(_, Some(_)) => { + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); let ptr_layout = cx.layout_of(ptr_ty); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); @@ -350,7 +364,9 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { continue; } PassMode::Cast(cast) => cast.llvm_type(cx), - PassMode::Indirect(_, None) => cx.type_ptr_to(arg.memory_ty(cx)), + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + cx.type_ptr_to(arg.memory_ty(cx)) + } }; llargument_tys.push(llarg_ty); } @@ -402,35 +418,54 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } let mut i = 0; - let mut apply = |attrs: &ArgAttributes, ty: Option<&Type>| { - attrs.apply_llfn(llvm::AttributePlace::Argument(i), llfn, ty); + let mut apply = |attrs: &ArgAttributes| { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), llfn); i += 1; + i - 1 }; match self.ret.mode { PassMode::Direct(ref attrs) => { - attrs.apply_llfn(llvm::AttributePlace::ReturnValue, llfn, None); + attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, llfn); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(attrs); + llvm::Attribute::StructRet.apply_llfn(llvm::AttributePlace::Argument(i), llfn); } - PassMode::Indirect(ref attrs, _) => apply(attrs, Some(self.ret.layout.llvm_type(cx))), _ => {} } for arg in &self.args { if arg.pad.is_some() { - apply(&ArgAttributes::new(), None); + apply(&ArgAttributes::new()); } match arg.mode { PassMode::Ignore => {} - PassMode::Direct(ref attrs) | PassMode::Indirect(ref attrs, None) => { - apply(attrs, Some(arg.layout.llvm_type(cx))) + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(attrs); + unsafe { + llvm::LLVMRustAddByValAttr( + llfn, + llvm::AttributePlace::Argument(i).as_uint(), + arg.layout.llvm_type(cx), + ); + } } - PassMode::Indirect(ref attrs, Some(ref extra_attrs)) => { - apply(attrs, None); - apply(extra_attrs, None); + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(attrs); + } + PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + assert!(!on_stack); + apply(attrs); + apply(extra_attrs); } PassMode::Pair(ref a, ref b) => { - apply(a, None); - apply(b, None); + apply(a); + apply(b); + } + PassMode::Cast(_) => { + apply(&ArgAttributes::new()); } - PassMode::Cast(_) => apply(&ArgAttributes::new(), None), } } } @@ -439,15 +474,21 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite. let mut i = 0; - let mut apply = |attrs: &ArgAttributes, ty: Option<&Type>| { - attrs.apply_callsite(llvm::AttributePlace::Argument(i), callsite, ty); + let mut apply = |attrs: &ArgAttributes| { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), callsite); i += 1; + i - 1 }; match self.ret.mode { PassMode::Direct(ref attrs) => { - attrs.apply_callsite(llvm::AttributePlace::ReturnValue, callsite, None); + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, callsite); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(attrs); + llvm::Attribute::StructRet + .apply_callsite(llvm::AttributePlace::Argument(i), callsite); } - PassMode::Indirect(ref attrs, _) => apply(attrs, Some(self.ret.layout.llvm_type(bx))), _ => {} } if let abi::Abi::Scalar(ref scalar) = self.ret.layout.abi { @@ -465,22 +506,39 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } for arg in &self.args { if arg.pad.is_some() { - apply(&ArgAttributes::new(), None); + apply(&ArgAttributes::new()); } match arg.mode { PassMode::Ignore => {} - PassMode::Direct(ref attrs) | PassMode::Indirect(ref attrs, None) => { - apply(attrs, Some(arg.layout.llvm_type(bx))) + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(attrs); + unsafe { + llvm::LLVMRustAddByValCallSiteAttr( + callsite, + llvm::AttributePlace::Argument(i).as_uint(), + arg.layout.llvm_type(bx), + ); + } } - PassMode::Indirect(ref attrs, Some(ref extra_attrs)) => { - apply(attrs, None); - apply(extra_attrs, None); + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(attrs); + } + PassMode::Indirect { + ref attrs, + extra_attrs: Some(ref extra_attrs), + on_stack: _, + } => { + apply(attrs); + apply(extra_attrs); } PassMode::Pair(ref a, ref b) => { - apply(a, None); - apply(b, None); + apply(a); + apply(b); + } + PassMode::Cast(_) => { + apply(&ArgAttributes::new()); } - PassMode::Cast(_) => apply(&ArgAttributes::new(), None), } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index b5d279eeb6f..8801211d51b 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -261,6 +261,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { InlineAsmArch::Hexagon => {} InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} InlineAsmArch::SpirV => {} + InlineAsmArch::Wasm32 => {} } } if !options.contains(InlineAsmOptions::NOMEM) { @@ -519,6 +520,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -584,6 +586,7 @@ fn modifier_to_llvm( _ => unreachable!(), }, InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } @@ -626,6 +629,7 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index e06e2d45665..97c38e04bc1 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -35,11 +35,7 @@ fn inline(cx: &CodegenCx<'ll, '_>, val: &'ll Value, inline: InlineAttr) { Attribute::NoInline.apply_llfn(Function, val); } } - None => { - Attribute::InlineHint.unapply_llfn(Function, val); - Attribute::AlwaysInline.unapply_llfn(Function, val); - Attribute::NoInline.unapply_llfn(Function, val); - } + None => {} }; } @@ -131,9 +127,6 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { return; } - // FIXME(richkadel): Make sure probestack plays nice with `-Z instrument-coverage` - // or disable it if not, similar to above early exits. - // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. llvm::AddFunctionAttrStringValue( @@ -229,12 +222,14 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: } } - // FIXME(eddyb) consolidate these two `inline` calls (and avoid overwrites). - if instance.def.requires_inline(cx.tcx) { - inline(cx, llfn, attributes::InlineAttr::Hint); - } - - inline(cx, llfn, codegen_fn_attrs.inline.clone()); + let inline_attr = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + InlineAttr::Never + } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; + inline(cx, llfn, inline_attr); // The `uwtable` attribute according to LLVM is: // diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 6f956c3bcc1..7407dfc455d 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -152,7 +152,8 @@ pub fn target_machine_factory( let features = features.join(","); let features = CString::new(features).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); - let trap_unreachable = sess.target.trap_unreachable; + let trap_unreachable = + sess.opts.debugging_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); let emit_stack_size_section = sess.opts.debugging_opts.emit_stack_sizes; let asm_comments = sess.asm_comments(); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 41827a91ba4..72ba5bbd5f2 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -3,11 +3,14 @@ use crate::coverageinfo; use crate::llvm; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; -use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods}; -use rustc_data_structures::fx::FxIndexSet; +use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, FunctionCoverage}; +use rustc_codegen_ssa::traits::ConstMethods; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; +use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE}; use rustc_llvm::RustString; use rustc_middle::mir::coverage::CodeRegion; +use rustc_middle::ty::{Instance, TyCtxt}; +use rustc_span::Symbol; use std::ffi::CString; @@ -15,9 +18,9 @@ use tracing::debug; /// Generates and exports the Coverage Map. /// -/// This Coverage Map complies with Coverage Mapping Format version 3 (zero-based encoded as 2), -/// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/llvmorg-8.0.0/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format) -/// and published in Rust's current (July 2020) fork of LLVM. This version is supported by the +/// This Coverage Map complies with Coverage Mapping Format version 4 (zero-based encoded as 3), +/// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format) +/// and published in Rust's current (November 2020) fork of LLVM. This version is supported by the /// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM. /// /// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with @@ -26,7 +29,17 @@ use tracing::debug; /// undocumented details in Clang's implementation (that may or may not be important) were also /// replicated for Rust's Coverage Map. pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { - let function_coverage_map = match cx.coverage_context() { + let tcx = cx.tcx; + // Ensure LLVM supports Coverage Map Version 4 (encoded as a zero-based value: 3). + // If not, the LLVM Version must be less than 11. + let version = coverageinfo::mapping_version(); + if version != 3 { + tcx.sess.fatal("rustc option `-Z instrument-coverage` requires LLVM 11 or higher."); + } + + debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); + + let mut function_coverage_map = match cx.coverage_context() { Some(ctx) => ctx.take_function_coverage_map(), None => return, }; @@ -35,49 +48,54 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { return; } + add_unreachable_coverage(tcx, &mut function_coverage_map); + let mut mapgen = CoverageMapGenerator::new(); // Encode coverage mappings and generate function records - let mut function_records = Vec::<&'ll llvm::Value>::new(); - let coverage_mappings_buffer = llvm::build_byte_buffer(|coverage_mappings_buffer| { - for (instance, function_coverage) in function_coverage_map.into_iter() { - debug!("Generate coverage map for: {:?}", instance); - - let mangled_function_name = cx.tcx.symbol_name(instance).to_string(); - let function_source_hash = function_coverage.source_hash(); - let (expressions, counter_regions) = - function_coverage.get_expressions_and_counter_regions(); - - let old_len = coverage_mappings_buffer.len(); - mapgen.write_coverage_mappings(expressions, counter_regions, coverage_mappings_buffer); - let mapping_data_size = coverage_mappings_buffer.len() - old_len; - debug_assert!( - mapping_data_size > 0, - "Every `FunctionCoverage` should have at least one counter" - ); - - let function_record = mapgen.make_function_record( - cx, - mangled_function_name, - function_source_hash, - mapping_data_size, - ); - function_records.push(function_record); - } - }); + let mut function_data = Vec::new(); + for (instance, function_coverage) in function_coverage_map { + debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); + let mangled_function_name = tcx.symbol_name(instance).to_string(); + let function_source_hash = function_coverage.source_hash(); + let (expressions, counter_regions) = + function_coverage.get_expressions_and_counter_regions(); + + let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { + mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); + }); + debug_assert!( + coverage_mapping_buffer.len() > 0, + "Every `FunctionCoverage` should have at least one counter" + ); + + function_data.push((mangled_function_name, function_source_hash, coverage_mapping_buffer)); + } // Encode all filenames referenced by counters/expressions in this module let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer); }); + let filenames_size = filenames_buffer.len(); + let filenames_val = cx.const_bytes(&filenames_buffer[..]); + let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); + // Generate the LLVM IR representation of the coverage map and store it in a well-known global - mapgen.save_generated_coverage_map( - cx, - function_records, - filenames_buffer, - coverage_mappings_buffer, - ); + let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); + + for (mangled_function_name, function_source_hash, coverage_mapping_buffer) in function_data { + save_function_record( + cx, + mangled_function_name, + function_source_hash, + filenames_ref, + coverage_mapping_buffer, + ); + } + + // Save the coverage data value to LLVM IR + coverageinfo::save_cov_data_to_mod(cx, cov_data_val); } struct CoverageMapGenerator { @@ -92,12 +110,12 @@ impl CoverageMapGenerator { /// Using the `expressions` and `counter_regions` collected for the current function, generate /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into - /// the given `coverage_mappings` byte buffer, compliant with the LLVM Coverage Mapping format. - fn write_coverage_mappings( + /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format. + fn write_coverage_mapping( &mut self, expressions: Vec<CounterExpression>, counter_regions: impl Iterator<Item = (Counter, &'a CodeRegion)>, - coverage_mappings_buffer: &RustString, + coverage_mapping_buffer: &RustString, ) { let mut counter_regions = counter_regions.collect::<Vec<_>>(); if counter_regions.is_empty() { @@ -145,89 +163,228 @@ impl CoverageMapGenerator { virtual_file_mapping, expressions, mapping_regions, - coverage_mappings_buffer, + coverage_mapping_buffer, ); } - /// Generate and return the function record `Value` - fn make_function_record( - &mut self, - cx: &CodegenCx<'ll, 'tcx>, - mangled_function_name: String, - function_source_hash: u64, - mapping_data_size: usize, - ) -> &'ll llvm::Value { - let name_ref = coverageinfo::compute_hash(&mangled_function_name); - let name_ref_val = cx.const_u64(name_ref); - let mapping_data_size_val = cx.const_u32(mapping_data_size as u32); - let func_hash_val = cx.const_u64(function_source_hash); - cx.const_struct( - &[name_ref_val, mapping_data_size_val, func_hash_val], - /*packed=*/ true, - ) - } - - /// Combine the filenames and coverage mappings buffers, construct coverage map header and the - /// array of function records, and combine everything into the complete coverage map. Save the - /// coverage map data into the LLVM IR as a static global using a specific, well-known section - /// and name. - fn save_generated_coverage_map( + /// Construct coverage map header and the array of function records, and combine them into the + /// coverage map. Save the coverage map data into the LLVM IR as a static global using a + /// specific, well-known section and name. + fn generate_coverage_map( self, cx: &CodegenCx<'ll, 'tcx>, - function_records: Vec<&'ll llvm::Value>, - filenames_buffer: Vec<u8>, - mut coverage_mappings_buffer: Vec<u8>, - ) { - // Concatenate the encoded filenames and encoded coverage mappings, and add additional zero - // bytes as-needed to ensure 8-byte alignment. - let mut coverage_size = coverage_mappings_buffer.len(); - let filenames_size = filenames_buffer.len(); - let remaining_bytes = - (filenames_size + coverage_size) % coverageinfo::COVMAP_VAR_ALIGN_BYTES; - if remaining_bytes > 0 { - let pad = coverageinfo::COVMAP_VAR_ALIGN_BYTES - remaining_bytes; - coverage_mappings_buffer.append(&mut [0].repeat(pad)); - coverage_size += pad; - } - let filenames_and_coverage_mappings = [filenames_buffer, coverage_mappings_buffer].concat(); - let filenames_and_coverage_mappings_val = - cx.const_bytes(&filenames_and_coverage_mappings[..]); - - debug!( - "cov map: n_records = {}, filenames_size = {}, coverage_size = {}, 0-based version = {}", - function_records.len(), - filenames_size, - coverage_size, - coverageinfo::mapping_version() - ); + version: u32, + filenames_size: usize, + filenames_val: &'ll llvm::Value, + ) -> &'ll llvm::Value { + debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); - // Create the coverage data header - let n_records_val = cx.const_u32(function_records.len() as u32); + // Create the coverage data header (Note, fields 0 and 2 are now always zero, + // as of `llvm::coverage::CovMapVersion::Version4`.) + let zero_was_n_records_val = cx.const_u32(0); let filenames_size_val = cx.const_u32(filenames_size as u32); - let coverage_size_val = cx.const_u32(coverage_size as u32); - let version_val = cx.const_u32(coverageinfo::mapping_version()); + let zero_was_coverage_size_val = cx.const_u32(0); + let version_val = cx.const_u32(version); let cov_data_header_val = cx.const_struct( - &[n_records_val, filenames_size_val, coverage_size_val, version_val], + &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], /*packed=*/ false, ); - // Create the function records array - let name_ref_from_u64 = cx.type_i64(); - let mapping_data_size_from_u32 = cx.type_i32(); - let func_hash_from_u64 = cx.type_i64(); - let function_record_ty = cx.type_struct( - &[name_ref_from_u64, mapping_data_size_from_u32, func_hash_from_u64], - /*packed=*/ true, - ); - let function_records_val = cx.const_array(function_record_ty, &function_records[..]); - // Create the complete LLVM coverage data value to add to the LLVM IR - let cov_data_val = cx.const_struct( - &[cov_data_header_val, function_records_val, filenames_and_coverage_mappings_val], - /*packed=*/ false, - ); + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + } +} + +/// Construct a function record and combine it with the function's coverage mapping data. +/// Save the function record into the LLVM IR as a static global using a +/// specific, well-known section and name. +fn save_function_record( + cx: &CodegenCx<'ll, 'tcx>, + mangled_function_name: String, + function_source_hash: u64, + filenames_ref: u64, + coverage_mapping_buffer: Vec<u8>, +) { + // Concatenate the encoded coverage mappings + let coverage_mapping_size = coverage_mapping_buffer.len(); + let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer[..]); + + let func_name_hash = coverageinfo::hash_str(&mangled_function_name); + let func_name_hash_val = cx.const_u64(func_name_hash); + let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); + let func_hash_val = cx.const_u64(function_source_hash); + let filenames_ref_val = cx.const_u64(filenames_ref); + let func_record_val = cx.const_struct( + &[ + func_name_hash_val, + coverage_mapping_size_val, + func_hash_val, + filenames_ref_val, + coverage_mapping_val, + ], + /*packed=*/ true, + ); - // Save the coverage data value to LLVM IR - coverageinfo::save_map_to_mod(cx, cov_data_val); + // At the present time, the coverage map for Rust assumes every instrumented function `is_used`. + // Note that Clang marks functions as "unused" in `CodeGenPGO::emitEmptyCounterMapping`. (See: + // https://github.com/rust-lang/llvm-project/blob/de02a75e398415bad4df27b4547c25b896c8bf3b/clang%2Flib%2FCodeGen%2FCodeGenPGO.cpp#L877-L878 + // for example.) + // + // It's not yet clear if or how this may be applied to Rust in the future, but the `is_used` + // argument is available and handled similarly. + let is_used = true; + coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); +} + +/// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for +/// the functions that went through codegen; such as public functions and "used" functions +/// (functions referenced by other "used" or public items). Any other functions considered unused, +/// or "Unreachable" were still parsed and processed through the MIR stage. +/// +/// We can find the unreachable functions by the set difference of all MIR `DefId`s (`tcx` query +/// `mir_keys`) minus the codegenned `DefId`s (`tcx` query `collect_and_partition_mono_items`). +/// +/// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and +/// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`) +/// allocated to only one of those CGUs. We must NOT inject any "Unreachable" functions's +/// `CodeRegion`s more than once, so we have to pick which CGU's `function_coverage_map` to add +/// each "Unreachable" function to. +/// +/// Some constraints: +/// +/// 1. The file name of an "Unreachable" function must match the file name of the existing +/// codegenned (covered) function to which the unreachable code regions will be added. +/// 2. The function to which the unreachable code regions will be added must not be a genaric +/// function (must not have type parameters) because the coverage tools will get confused +/// if the codegenned function has more than one instantiation and additional `CodeRegion`s +/// attached to only one of those instantiations. +fn add_unreachable_coverage<'tcx>( + tcx: TyCtxt<'tcx>, + function_coverage_map: &mut FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>, +) { + // FIXME(#79622): Can this solution be simplified and/or improved? Are there other sources + // of compiler state data that might help (or better sources that could be exposed, but + // aren't yet)? + + // Note: If the crate *only* defines generic functions, there are no codegenerated non-generic + // functions to add any unreachable code to. In this case, the unreachable code regions will + // have no coverage, instead of having coverage with zero executions. + // + // This is probably still an improvement over Clang, which does not generate any coverage + // for uninstantiated template functions. + + let has_non_generic_def_ids = + function_coverage_map.keys().any(|instance| instance.def.attrs(tcx).len() == 0); + + if !has_non_generic_def_ids { + // There are no non-generic functions to add unreachable `CodeRegion`s to + return; + } + + let all_def_ids: DefIdSet = + tcx.mir_keys(LOCAL_CRATE).iter().map(|local_def_id| local_def_id.to_def_id()).collect(); + + let (codegenned_def_ids, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + + let mut unreachable_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default(); + for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) { + // Make sure the non-codegenned (unreachable) function has a file_name + if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) { + let def_ids = unreachable_def_ids_by_file + .entry(*non_codegenned_file_name) + .or_insert_with(|| Vec::new()); + def_ids.push(non_codegenned_def_id); + } + } + + if unreachable_def_ids_by_file.is_empty() { + // There are no unreachable functions with file names to add (in any CGU) + return; + } + + // Since there may be multiple `CodegenUnit`s, some codegenned_def_ids may be codegenned in a + // different CGU, and will be added to the function_coverage_map for each CGU. Determine which + // function_coverage_map has the responsibility for publishing unreachable coverage + // based on file name: + // + // For each covered file name, sort ONLY the non-generic codegenned_def_ids, and if + // covered_def_ids.contains(the first def_id) for a given file_name, add the unreachable code + // region in this function_coverage_map. Otherwise, ignore it and assume another CGU's + // function_coverage_map will be adding it (because it will be first for one, and only one, + // of them). + let mut sorted_codegenned_def_ids: Vec<DefId> = + codegenned_def_ids.iter().map(|def_id| *def_id).collect(); + sorted_codegenned_def_ids.sort_unstable(); + + let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default(); + for &def_id in sorted_codegenned_def_ids.iter() { + // Only consider non-generic functions, to potentially add unreachable code regions + if tcx.generics_of(def_id).count() == 0 { + if let Some(covered_file_name) = tcx.covered_file_name(def_id) { + // Only add files known to have unreachable functions + if unreachable_def_ids_by_file.contains_key(covered_file_name) { + first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id); + } + } + } + } + + // Get the set of def_ids with coverage regions, known by *this* CoverageContext. + let cgu_covered_def_ids: DefIdSet = + function_coverage_map.keys().map(|instance| instance.def.def_id()).collect(); + + let mut cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file + .iter() + .filter_map( + |(&file_name, def_id)| { + if cgu_covered_def_ids.contains(def_id) { Some(file_name) } else { None } + }, + ) + .collect(); + + // Find the first covered, non-generic function (instance) for each cgu_covered_file. Take the + // unreachable code regions for that file, and add them to the function. + // + // There are three `for` loops here, but (a) the lists have already been reduced to the minimum + // required values, the lists are further reduced (by `remove()` calls) when elements are no + // longer needed, and there are several opportunities to branch out of loops early. + for (instance, function_coverage) in function_coverage_map.iter_mut() { + if instance.def.attrs(tcx).len() > 0 { + continue; + } + // The covered function is not generic... + let covered_def_id = instance.def.def_id(); + if let Some(covered_file_name) = tcx.covered_file_name(covered_def_id) { + if !cgu_covered_files.remove(&covered_file_name) { + continue; + } + // The covered function's file is one of the files with unreachable code regions, so + // all of the unreachable code regions for this file will be added to this function. + for def_id in + unreachable_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() + { + // Note, this loop adds an unreachable code regions for each MIR-derived region. + // Alternatively, we could add a single code region for the maximum span of all + // code regions here. + // + // Observed downsides of this approach are: + // + // 1. The coverage results will appear inconsistent compared with the same (or + // similar) code in a function that is reached. + // 2. If the function is unreachable from one crate but reachable when compiling + // another referencing crate (such as a cross-crate reference to a + // generic function or inlined function), actual coverage regions overlaid + // on a single larger code span of `Zero` coverage can appear confusing or + // wrong. Chaning the unreachable coverage from a `code_region` to a + // `gap_region` can help, but still can look odd with `0` line counts for + // lines between executed (> 0) lines (such as for blank lines or comments). + for ®ion in tcx.covered_code_regions(def_id) { + function_coverage.add_unreachable_region(region.clone()); + } + } + if cgu_covered_files.is_empty() { + break; + } + } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index e21e03822eb..e777f363eb0 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -23,7 +23,7 @@ use tracing::debug; pub mod mapgen; -const COVMAP_VAR_ALIGN_BYTES: usize = 8; +const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'tcx> { @@ -177,17 +177,20 @@ pub(crate) fn write_mapping_to_buffer( ); } } +pub(crate) fn hash_str(strval: &str) -> u64 { + let strval = CString::new(strval).expect("null error converting hashable str to C string"); + unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } +} -pub(crate) fn compute_hash(name: &str) -> u64 { - let name = CString::new(name).expect("null error converting hashable name to C string"); - unsafe { llvm::LLVMRustCoverageComputeHash(name.as_ptr()) } +pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { + unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } } pub(crate) fn mapping_version() -> u32 { unsafe { llvm::LLVMRustCoverageMappingVersion() } } -pub(crate) fn save_map_to_mod<'ll, 'tcx>( +pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, cov_data_val: &'ll llvm::Value, ) { @@ -198,7 +201,7 @@ pub(crate) fn save_map_to_mod<'ll, 'tcx>( debug!("covmap var name: {:?}", covmap_var_name); let covmap_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteSectionNameToString(cx.llmod, s); + llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); }) .expect("Rust Coverage section name failed UTF-8 conversion"); debug!("covmap section name: {:?}", covmap_section_name); @@ -206,8 +209,43 @@ pub(crate) fn save_map_to_mod<'ll, 'tcx>( let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); llvm::set_initializer(llglobal, cov_data_val); llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::InternalLinkage); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::set_section(llglobal, &covmap_section_name); - llvm::set_alignment(llglobal, COVMAP_VAR_ALIGN_BYTES); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + cx.add_used_global(llglobal); +} + +pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + func_name_hash: u64, + func_record_val: &'ll llvm::Value, + is_used: bool, +) { + // Assign a name to the function record. This is used to merge duplicates. + // + // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that + // are included-but-not-used. If (or when) Rust generates functions that are + // included-but-not-used, note that a dummy description for a function included-but-not-used + // in a Crate can be replaced by full description provided by a different Crate. The two kinds + // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by + // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + let func_record_var_name = + format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + debug!("function record var name: {:?}", func_record_var_name); + + let func_record_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage function record section name failed UTF-8 conversion"); + debug!("function record section name: {:?}", func_record_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, &func_record_section_name); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); cx.add_used_global(llglobal); } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5e8ff14f0aa..96484034da7 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1152,10 +1152,7 @@ impl<'ll> MemberDescription<'ll> { self.size.bits(), self.align.bits() as u32, self.offset.bits(), - match self.discriminant { - None => None, - Some(value) => Some(cx.const_u64(value)), - }, + self.discriminant.map(|v| cx.const_u64(v)), self.flags, self.type_metadata, ) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index ac423d01bf1..bf0d499e6c4 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -792,7 +792,7 @@ fn generic_simd_intrinsic( _ => return_error!("`{}` is not an integral type", in_ty), }; require_simd!(arg_tys[1], "argument"); - let v_len = arg_tys[1].simd_size(tcx); + let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); require!( // Allow masks for vectors with fewer than 8 elements to be // represented with a u8 or i8. @@ -812,8 +812,6 @@ fn generic_simd_intrinsic( // every intrinsic below takes a SIMD vector as its first argument require_simd!(arg_tys[0], "input"); let in_ty = arg_tys[0]; - let in_elem = arg_tys[0].simd_type(tcx); - let in_len = arg_tys[0].simd_size(tcx); let comparison = match name { sym::simd_eq => Some(hir::BinOpKind::Eq), @@ -825,14 +823,15 @@ fn generic_simd_intrinsic( _ => None, }; + let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); if let Some(cmp_op) = comparison { require_simd!(ret_ty, "return"); - let out_len = ret_ty.simd_size(tcx); + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, "expected return type with length {} (same as input type `{}`), \ - found `{}` with length {}", + found `{}` with length {}", in_len, in_ty, ret_ty, @@ -842,7 +841,7 @@ fn generic_simd_intrinsic( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, "expected return type with integer elements, found `{}` with non-integer `{}`", ret_ty, - ret_ty.simd_type(tcx) + out_ty ); return Ok(compare_simd_types( @@ -855,14 +854,14 @@ fn generic_simd_intrinsic( )); } - if name_str.starts_with("simd_shuffle") { - let n: u64 = name_str["simd_shuffle".len()..].parse().unwrap_or_else(|_| { + if let Some(stripped) = name_str.strip_prefix("simd_shuffle") { + let n: u64 = stripped.parse().unwrap_or_else(|_| { span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") }); require_simd!(ret_ty, "return"); - let out_len = ret_ty.simd_size(tcx); + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); require!( out_len == n, "expected return type of length {}, found `{}` with length {}", @@ -871,13 +870,13 @@ fn generic_simd_intrinsic( out_len ); require!( - in_elem == ret_ty.simd_type(tcx), + in_elem == out_ty, "expected return element type `{}` (element of input `{}`), \ - found `{}` with element type `{}`", + found `{}` with element type `{}`", in_elem, in_ty, ret_ty, - ret_ty.simd_type(tcx) + out_ty ); let total_len = u128::from(in_len) * 2; @@ -946,7 +945,7 @@ fn generic_simd_intrinsic( let m_elem_ty = in_elem; let m_len = in_len; require_simd!(arg_tys[1], "argument"); - let v_len = arg_tys[1].simd_size(tcx); + let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); require!( m_len == v_len, "mismatched lengths: mask length `{}` != other vector length `{}`", @@ -1173,25 +1172,27 @@ fn generic_simd_intrinsic( require_simd!(ret_ty, "return"); // Of the same length: + let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( - in_len == arg_tys[1].simd_size(tcx), + in_len == out_len, "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", + found `{}` with length {}", "second", in_len, in_ty, arg_tys[1], - arg_tys[1].simd_size(tcx) + out_len ); require!( - in_len == arg_tys[2].simd_size(tcx), + in_len == out_len2, "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", + found `{}` with length {}", "third", in_len, in_ty, arg_tys[2], - arg_tys[2].simd_size(tcx) + out_len2 ); // The return type must match the first argument type @@ -1215,39 +1216,40 @@ fn generic_simd_intrinsic( // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument - let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind() { - ty::RawPtr(p) if p.ty == in_elem => { - (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) - } + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { + ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)), _ => { require!( false, "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*_ {}`", - arg_tys[1].simd_type(tcx), + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*_ {}`", + element_ty1, arg_tys[1], in_elem, in_ty, - arg_tys[1].simd_type(tcx), + element_ty1, in_elem ); unreachable!(); } }; assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx))); - assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx))); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); // The element type of the third argument must be a signed integer type of any width: - match arg_tys[2].simd_type(tcx).kind() { + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + match element_ty2.kind() { ty::Int(_) => (), _ => { require!( false, "expected element type `{}` of third argument `{}` \ to be a signed integer type", - arg_tys[2].simd_type(tcx), + element_ty2, arg_tys[2] ); } @@ -1299,25 +1301,27 @@ fn generic_simd_intrinsic( require_simd!(arg_tys[2], "third"); // Of the same length: + let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( - in_len == arg_tys[1].simd_size(tcx), + in_len == element_len1, "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", + found `{}` with length {}", "second", in_len, in_ty, arg_tys[1], - arg_tys[1].simd_size(tcx) + element_len1 ); require!( - in_len == arg_tys[2].simd_size(tcx), + in_len == element_len2, "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", + found `{}` with length {}", "third", in_len, in_ty, arg_tys[2], - arg_tys[2].simd_size(tcx) + element_len2 ); // This counts how many pointers @@ -1338,39 +1342,42 @@ fn generic_simd_intrinsic( // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument - let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind() { + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { - (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) + (ptr_count(element_ty1), non_ptr(element_ty1)) } _ => { require!( false, "expected element type `{}` of second argument `{}` \ - to be a pointer to the element type `{}` of the first \ - argument `{}`, found `{}` != `*mut {}`", - arg_tys[1].simd_type(tcx), + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*mut {}`", + element_ty1, arg_tys[1], in_elem, in_ty, - arg_tys[1].simd_type(tcx), + element_ty1, in_elem ); unreachable!(); } }; assert!(pointer_count > 0); - assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx))); - assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx))); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); // The element type of the third argument must be a signed integer type of any width: - match arg_tys[2].simd_type(tcx).kind() { + match element_ty2.kind() { ty::Int(_) => (), _ => { require!( false, "expected element type `{}` of third argument `{}` \ - to be a signed integer type", - arg_tys[2].simd_type(tcx), + be a signed integer type", + element_ty2, arg_tys[2] ); } @@ -1567,7 +1574,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, if name == sym::simd_cast { require_simd!(ret_ty, "return"); - let out_len = ret_ty.simd_size(tcx); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( in_len == out_len, "expected return type with length {} (same as input type `{}`), \ @@ -1578,8 +1585,6 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, out_len ); // casting cares about nominal type, not just structural type - let out_elem = ret_ty.simd_type(tcx); - if in_elem == out_elem { return Ok(args[0].immediate()); } @@ -1695,7 +1700,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, return_error!( "expected element type `{}` of vector type `{}` \ to be a signed or unsigned integer type", - arg_tys[0].simd_type(tcx), + arg_tys[0].simd_size_and_type(bx.tcx()).1, arg_tys[0] ); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 8b15c8b0eb6..41482d18946 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -642,7 +642,7 @@ pub type InlineAsmDiagHandler = unsafe extern "C" fn(&SMDiagnostic, *const c_voi pub mod coverageinfo { use super::coverage_map; - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/10.0-2020-05-05/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L205-L221) + /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L206-L222) #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum RegionKind { @@ -665,13 +665,13 @@ pub mod coverageinfo { /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the /// coverage map, in accordance with the - /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/llvmorg-8.0.0/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). + /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). /// The struct composes fields representing the `Counter` type and value(s) (injected counter /// ID, or expression type and operands), the source file (an indirect index into a "filenames /// array", encoded separately), and source location (start and end positions of the represented /// code region). /// - /// Aligns with [llvm::coverage::CounterMappingRegion](https://github.com/rust-lang/llvm-project/blob/rustc/10.0-2020-05-05/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L223-L226) + /// Aligns with [llvm::coverage::CounterMappingRegion](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L224-L227) /// Important: The Rust struct layout (order and types of fields) must match its C++ /// counterpart. #[derive(Copy, Clone, Debug)] @@ -1791,10 +1791,14 @@ extern "C" { pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &'a Value, FuncName: *const c_char) -> &'a Value; - pub fn LLVMRustCoverageComputeHash(Name: *const c_char) -> u64; + pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; + pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteSectionNameToString(M: &Module, Str: &RustString); + pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 53a404ee019..fc40065a966 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -220,12 +220,24 @@ pub fn set_linkage(llglobal: &Value, linkage: Linkage) { } } +pub fn set_visibility(llglobal: &Value, visibility: Visibility) { + unsafe { + LLVMRustSetVisibility(llglobal, visibility); + } +} + pub fn set_alignment(llglobal: &Value, bytes: usize) { unsafe { ffi::LLVMSetAlignment(llglobal, bytes as c_uint); } } +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { + unsafe { + LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + } +} + /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. pub fn get_param(llfn: &Value, index: c_uint) -> &Value { unsafe { diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index a8a1646183c..a3139ce5a34 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -129,6 +129,13 @@ pub fn time_trace_profiler_finish(file_name: &str) { // WARNING: the features after applying `to_llvm_feature` must be known // to LLVM or the feature detection code will walk past the end of the feature // array, leading to crashes. +// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def +// where the * matches the architecture's name +// Beware to not use the llvm github project for this, but check the git submodule +// found in src/llvm-project +// Though note that Rust can also be build with an external precompiled version of LLVM +// which might lead to failures if the oldest tested / supported LLVM version +// doesn't yet support the relevant intrinsics pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str { let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; match (arch, s) { @@ -136,6 +143,9 @@ pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str { ("x86", "rdrand") => "rdrnd", ("x86", "bmi1") => "bmi", ("x86", "cmpxchg16b") => "cx16", + ("x86", "avx512vaes") => "vaes", + ("x86", "avx512gfni") => "gfni", + ("x86", "avx512vpclmulqdq") => "vpclmulqdq", ("aarch64", "fp") => "fp-armv8", ("aarch64", "fp16") => "fullfp16", (_, s) => s, |