diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
20 files changed, 717 insertions, 478 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index d9393ffe534..854e3ccc21b 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,6 +1,7 @@ use crate::builder::Builder; use crate::context::CodegenCx; use crate::llvm::{self, AttributePlace}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -41,12 +42,29 @@ impl ArgAttributeExt for ArgAttribute { } pub trait ArgAttributesExt { - fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value); - fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value); + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ); +} + +fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { + // LLVM prior to version 12 has known miscompiles in the presence of + // noalias attributes (see #54878). Only enable mutable noalias by + // default for versions we believe to be safe. + cx.tcx + .sess + .opts + .debugging_opts + .mutable_noalias + .unwrap_or_else(|| llvm_util::get_version() >= (12, 0, 0)) } impl ArgAttributesExt for ArgAttributes { - fn apply_attrs_to_llfn(&self, idx: AttributePlace, llfn: &Value) { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { let mut regular = self.regular; unsafe { let deref = self.pointee_size.bytes(); @@ -62,6 +80,9 @@ impl ArgAttributesExt for ArgAttributes { llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); } regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); + if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { + llvm::Attribute::NoAlias.apply_llfn(idx, llfn); + } match self.arg_ext { ArgExtension::None => {} ArgExtension::Zext => { @@ -74,7 +95,12 @@ impl ArgAttributesExt for ArgAttributes { } } - fn apply_attrs_to_callsite(&self, idx: AttributePlace, callsite: &Value) { + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ) { let mut regular = self.regular; unsafe { let deref = self.pointee_size.bytes(); @@ -98,6 +124,9 @@ impl ArgAttributesExt for ArgAttributes { ); } regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); + if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { + llvm::Attribute::NoAlias.apply_callsite(idx, callsite); + } match self.arg_ext { ArgExtension::None => {} ArgExtension::Zext => { @@ -419,13 +448,13 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { let mut i = 0; let mut apply = |attrs: &ArgAttributes| { - attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), llfn); + attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); i += 1; i - 1 }; match self.ret.mode { PassMode::Direct(ref attrs) => { - attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, llfn); + attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); } PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); @@ -480,18 +509,18 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite. let mut i = 0; - let mut apply = |attrs: &ArgAttributes| { - attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), callsite); + let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); i += 1; i - 1 }; match self.ret.mode { PassMode::Direct(ref attrs) => { - attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, callsite); + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, &bx.cx, callsite); } PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { assert!(!on_stack); - let i = apply(attrs); + let i = apply(bx.cx, attrs); unsafe { llvm::LLVMRustAddStructRetCallSiteAttr( callsite, @@ -517,12 +546,12 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } for arg in &self.args { if arg.pad.is_some() { - apply(&ArgAttributes::new()); + apply(bx.cx, &ArgAttributes::new()); } match arg.mode { PassMode::Ignore => {} PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { - let i = apply(attrs); + let i = apply(bx.cx, attrs); unsafe { llvm::LLVMRustAddByValCallSiteAttr( callsite, @@ -533,22 +562,22 @@ impl<'tcx> FnAbiLlvmExt<'tcx> for FnAbi<'tcx, Ty<'tcx>> { } PassMode::Direct(ref attrs) | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { - apply(attrs); + apply(bx.cx, attrs); } PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack: _, } => { - apply(attrs); - apply(extra_attrs); + apply(bx.cx, attrs); + apply(bx.cx, extra_attrs); } PassMode::Pair(ref a, ref b) => { - apply(a); - apply(b); + apply(bx.cx, a); + apply(bx.cx, b); } PassMode::Cast(_) => { - apply(&ArgAttributes::new()); + apply(bx.cx, &ArgAttributes::new()); } } } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 38c8ae711a4..84b091d8d4d 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -14,7 +14,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::{bug, span_bug}; -use rustc_span::{Pos, Span}; +use rustc_span::{Pos, Span, Symbol}; use rustc_target::abi::*; use rustc_target::asm::*; @@ -125,15 +125,39 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { // Collect the types of output operands let mut constraints = vec![]; + let mut clobbers = vec![]; let mut output_types = vec![]; let mut op_idx = FxHashMap::default(); for (idx, op) in operands.iter().enumerate() { match *op { InlineAsmOperandRef::Out { reg, late, place } => { + let is_target_supported = |reg_class: InlineAsmRegClass| { + for &(_, feature) in reg_class.supported_types(asm_arch) { + if let Some(feature) = feature { + if self.tcx.sess.target_features.contains(&Symbol::intern(feature)) + { + return true; + } + } else { + // Register class is unconditionally supported + return true; + } + } + false + }; + let mut layout = None; let ty = if let Some(ref place) = place { layout = Some(&place.layout); llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) + } else if !is_target_supported(reg.reg_class()) { + // We turn discarded outputs into clobber constraints + // if the target feature needed by the register class is + // disabled. This is necessary otherwise LLVM will try + // to actually allocate a register for the dummy output. + assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); + clobbers.push(format!("~{}", reg_to_llvm(reg, None))); + continue; } else { // If the output is discarded, we don't really care what // type is used. We're just using this to tell LLVM to @@ -244,6 +268,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } + constraints.append(&mut clobbers); if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { match asm_arch { InlineAsmArch::AArch64 | InlineAsmArch::Arm => { @@ -528,6 +553,7 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::Err => unreachable!(), } .to_string(), } @@ -594,6 +620,7 @@ fn modifier_to_llvm( InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::Err => unreachable!(), } } @@ -637,6 +664,7 @@ fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { bug!("LLVM backend does not support SPIR-V") } + InlineAsmRegClass::Err => unreachable!(), } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 09ece6164eb..e06c1c825f6 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -11,9 +11,9 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt}; -use rustc_session::config::{OptLevel, SanitizerSet}; +use rustc_session::config::OptLevel; use rustc_session::Session; -use rustc_target::spec::StackProbeType; +use rustc_target::spec::{SanitizerSet, StackProbeType}; use crate::attributes; use crate::llvm::AttributePlace::Function; @@ -152,18 +152,6 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { } } -pub fn llvm_target_features(sess: &Session) -> impl Iterator<Item = &str> { - const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; - - let cmdline = sess - .opts - .cg - .target_feature - .split(',') - .filter(|f| !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s))); - sess.target.features.split(',').chain(cmdline).filter(|l| !l.is_empty()) -} - pub fn apply_target_cpu_attr(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { let target_cpu = SmallCStr::new(llvm_util::target_cpu(cx.tcx.sess)); llvm::AddFunctionAttrStringValue( @@ -266,6 +254,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: attributes::emit_uwtable(llfn, true); } + // FIXME: none of these three functions interact with source level attributes. set_frame_pointer_elimination(cx, llfn); set_instrument_function(cx, llfn); set_probestack(cx, llfn); @@ -291,6 +280,9 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { llvm::AddFunctionAttrString(llfn, Function, cstr!("cmse_nonsecure_entry")); } + if let Some(align) = codegen_fn_attrs.alignment { + llvm::set_alignment(llfn, align as usize); + } sanitize(cx, codegen_fn_attrs.no_sanitize, llfn); // Always annotate functions with the target-cpu they are compiled for. @@ -301,20 +293,22 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: // The target doesn't care; the subtarget reads our attribute. apply_tune_cpu_attr(cx, llfn); - let features = llvm_target_features(cx.tcx.sess) - .map(|s| s.to_string()) - .chain(codegen_fn_attrs.target_features.iter().map(|f| { + let function_features = codegen_fn_attrs + .target_features + .iter() + .map(|f| { let feature = &f.as_str(); format!("+{}", llvm_util::to_llvm_feature(cx.tcx.sess, feature)) - })) + }) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), })) - .collect::<Vec<String>>() - .join(","); - - if !features.is_empty() { + .collect::<Vec<String>>(); + if !function_features.is_empty() { + let mut global_features = llvm_util::llvm_global_features(cx.tcx.sess); + global_features.extend(function_features.into_iter()); + let features = global_features.join(","); let val = CString::new(features).unwrap(); llvm::AddFunctionAttrStringValue( llfn, @@ -327,7 +321,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: // Note that currently the `wasm-import-module` doesn't do anything, but // eventually LLVM 7 should read this and ferry the appropriate import // module to the output file. - if cx.tcx.sess.target.arch == "wasm32" { + if cx.tcx.sess.target.is_like_wasm { if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { llvm::AddFunctionAttrStringValue( llfn, diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 5effe687528..4226ed7d99b 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -24,6 +24,7 @@ use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; +use std::iter; use std::path::Path; use std::ptr; use std::slice; @@ -916,9 +917,7 @@ impl ThinLTOKeysMap { modules: &[llvm::ThinLTOModule], names: &[CString], ) -> Self { - let keys = modules - .iter() - .zip(names.iter()) + let keys = iter::zip(modules, names) .map(|(module, name)| { let key = build_string(|rust_str| unsafe { llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index c224da7885b..b628ae3ae3a 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,4 +1,3 @@ -use crate::attributes; use crate::back::lto::ThinBuffer; use crate::back::profiling::{ selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, @@ -24,11 +23,11 @@ use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{self, Lto, OutputType, Passes, SanitizerSet, SwitchWithOptPath}; +use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; -use rustc_target::spec::{CodeModel, RelocModel, SplitDebuginfo}; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; use tracing::debug; use libc::{c_char, c_int, c_uint, c_void, size_t}; @@ -140,7 +139,7 @@ fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { } } -fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel { +pub(crate) fn to_llvm_code_model(code_model: Option<CodeModel>) -> llvm::CodeModel { match code_model { Some(CodeModel::Tiny) => llvm::CodeModel::Tiny, Some(CodeModel::Small) => llvm::CodeModel::Small, @@ -166,23 +165,18 @@ pub fn target_machine_factory( let code_model = to_llvm_code_model(sess.code_model()); - let mut features = llvm_util::handle_native_features(sess); - features.extend(attributes::llvm_target_features(sess).map(|s| s.to_owned())); let mut singlethread = sess.target.singlethread; // On the wasm target once the `atomics` feature is enabled that means that // we're no longer single-threaded, or otherwise we don't want LLVM to // lower atomic operations to single-threaded operations. - if singlethread - && sess.target.llvm_target.contains("wasm32") - && sess.target_features.contains(&sym::atomics) - { + if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) { singlethread = false; } let triple = SmallCStr::new(&sess.target.llvm_target); let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); - let features = features.join(","); + let features = llvm_util::llvm_global_features(sess).join(","); let features = CString::new(features).unwrap(); let abi = SmallCStr::new(&sess.target.llvm_abiname); let trap_unreachable = @@ -551,6 +545,15 @@ pub(crate) unsafe fn optimize( llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); continue; } + if pass_name == "insert-gcov-profiling" || pass_name == "instrprof" { + // Instrumentation must be inserted before optimization, + // otherwise LLVM may optimize some functions away which + // breaks llvm-cov. + // + // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. + llvm::LLVMRustAddPass(mpm, find_pass(pass_name).unwrap()); + continue; + } if let Some(pass) = find_pass(pass_name) { extra_passes.push(pass); @@ -1044,7 +1047,7 @@ pub unsafe fn with_llvm_pmb( // thresholds copied from clang. match (opt_level, opt_size, inline_threshold) { (.., Some(t)) => { - llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t as u32); + llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, t); } (llvm::CodeGenOptLevel::Aggressive, ..) => { llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275); diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index d5be3132dee..6f6c649bb0b 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -32,8 +32,9 @@ use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::middle::exported_symbols; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{DebugInfo, SanitizerSet}; +use rustc_session::config::DebugInfo; use rustc_span::symbol::Symbol; +use rustc_target::spec::SanitizerSet; use std::ffi::CString; use std::time::Instant; @@ -143,7 +144,7 @@ pub fn compile_codegen_unit( // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.used` variable, created next. - if cx.sess().opts.debugging_opts.instrument_coverage { + if cx.sess().instrument_coverage() { cx.coverageinfo_finalize(); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index f4852c91e53..896e56a9a1e 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -21,6 +21,7 @@ use rustc_target::abi::{self, Align, Size}; use rustc_target::spec::{HasTargetSpec, Target}; use std::borrow::Cow; use std::ffi::CStr; +use std::iter; use std::ops::{Deref, Range}; use std::ptr; use tracing::debug; @@ -1352,18 +1353,14 @@ impl Builder<'a, 'll, 'tcx> { let param_tys = self.cx.func_params_types(fn_ty); - let all_args_match = param_tys - .iter() - .zip(args.iter().map(|&v| self.val_ty(v))) + let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.val_ty(v))) .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); if all_args_match { return Cow::Borrowed(args); } - let casted_args: Vec<_> = param_tys - .into_iter() - .zip(args.iter()) + let casted_args: Vec<_> = iter::zip(param_tys, args) .enumerate() .map(|(i, (expected_ty, &actual_val))| { let actual_ty = self.val_ty(actual_val); diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 367c1f4811c..b26969a5012 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -14,6 +14,7 @@ use tracing::debug; use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_target::spec::RelocModel; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -170,17 +171,19 @@ pub fn get_fn(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value } } } - } - // MinGW: For backward compatibility we rely on the linker to decide whether it - // should use dllimport for functions. - if cx.use_dll_storage_attrs - && tcx.is_dllimport_foreign_item(instance_def_id) - && tcx.sess.target.env != "gnu" - { - unsafe { + // MinGW: For backward compatibility we rely on the linker to decide whether it + // should use dllimport for functions. + if cx.use_dll_storage_attrs + && tcx.is_dllimport_foreign_item(instance_def_id) + && tcx.sess.target.env != "gnu" + { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } + + if cx.tcx.sess.relocation_model() == RelocModel::Static { + llvm::LLVMRustSetDSOLocal(llfn, true); + } } llfn diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 9f6a2ae3ca1..773c0c16328 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1,4 +1,5 @@ use crate::attributes; +use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; use crate::coverageinfo; use crate::debuginfo; @@ -78,7 +79,7 @@ pub struct CodegenCx<'ll, 'tcx> { pub pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>, pub isize_ty: &'ll Type, - pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'tcx>>, + pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CrateDebugContext<'ll, 'tcx>>, eh_personality: Cell<Option<&'ll Value>>, @@ -100,10 +101,6 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { } } -fn strip_x86_address_spaces(data_layout: String) -> String { - data_layout.replace("-p270:32:32-p271:32:32-p272:64:64-", "-") -} - fn strip_powerpc64_vectors(data_layout: String) -> String { data_layout.replace("-v256:256:256-v512:512:512", "") } @@ -118,11 +115,6 @@ pub unsafe fn create_module( let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); let mut target_data_layout = sess.target.data_layout.clone(); - if llvm_util::get_version() < (10, 0, 0) - && (sess.target.arch == "x86" || sess.target.arch == "x86_64") - { - target_data_layout = strip_x86_address_spaces(target_data_layout); - } if llvm_util::get_version() < (12, 0, 0) && sess.target.arch == "powerpc64" { target_data_layout = strip_powerpc64_vectors(target_data_layout); } @@ -181,6 +173,13 @@ pub unsafe fn create_module( } } + // Linking object files with different code models is undefined behavior + // because the compiler would have to generate additional code (to span + // longer jumps) if a larger code model is used with a smaller one. + // + // See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323. + llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model())); + // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { @@ -272,7 +271,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); - let coverage_cx = if tcx.sess.opts.debugging_opts.instrument_coverage { + let coverage_cx = if tcx.sess.instrument_coverage() { let covctx = coverageinfo::CrateCoverageContext::new(); Some(covctx) } else { @@ -323,7 +322,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } #[inline] - pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'tcx>> { + pub fn coverage_context(&'a self) -> Option<&'a coverageinfo::CrateCoverageContext<'ll, 'tcx>> { self.coverage_cx.as_ref() } } @@ -704,7 +703,7 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.va_end", fn(i8p) -> void); ifn!("llvm.va_copy", fn(i8p, i8p) -> void); - if self.sess().opts.debugging_opts.instrument_coverage { + if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 444a9d4ba04..2ac814bf228 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -3,13 +3,12 @@ use crate::coverageinfo; use crate::llvm; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression, FunctionCoverage}; -use rustc_codegen_ssa::traits::ConstMethods; +use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; +use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_hir::def_id::{DefId, DefIdSet, LOCAL_CRATE}; use rustc_llvm::RustString; use rustc_middle::mir::coverage::CodeRegion; -use rustc_middle::ty::{Instance, TyCtxt}; use rustc_span::Symbol; use std::ffi::CString; @@ -20,16 +19,17 @@ use tracing::debug; /// /// This Coverage Map complies with Coverage Mapping Format version 4 (zero-based encoded as 3), /// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/11.0-2020-10-12/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format) -/// and published in Rust's current (November 2020) fork of LLVM. This version is supported by the -/// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM. +/// and published in Rust's November 2020 fork of LLVM. This version is supported by the LLVM +/// coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM. /// /// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with -/// version 3. Clang's implementation of Coverage Map generation was referenced when implementing -/// this Rust version, and though the format documentation is very explicit and detailed, some -/// undocumented details in Clang's implementation (that may or may not be important) were also -/// replicated for Rust's Coverage Map. +/// the same version. Clang's implementation of Coverage Map generation was referenced when +/// implementing this Rust version, and though the format documentation is very explicit and +/// detailed, some undocumented details in Clang's implementation (that may or may not be important) +/// were also replicated for Rust's Coverage Map. pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { let tcx = cx.tcx; + // Ensure LLVM supports Coverage Map Version 4 (encoded as a zero-based value: 3). // If not, the LLVM Version must be less than 11. let version = coverageinfo::mapping_version(); @@ -39,17 +39,24 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); - let mut function_coverage_map = match cx.coverage_context() { + // In order to show that unused functions have coverage counts of zero (0), LLVM requires the + // functions exist. Generate synthetic functions with a (required) single counter, and add the + // MIR `Coverage` code regions to the `function_coverage_map`, before calling + // `ctx.take_function_coverage_map()`. + if !tcx.sess.instrument_coverage_except_unused_functions() { + add_unused_functions(cx); + } + + let function_coverage_map = match cx.coverage_context() { Some(ctx) => ctx.take_function_coverage_map(), None => return, }; + if function_coverage_map.is_empty() { // This module has no functions with coverage instrumentation return; } - add_unreachable_coverage(tcx, &mut function_coverage_map); - let mut mapgen = CoverageMapGenerator::new(); // Encode coverage mappings and generate function records @@ -57,7 +64,8 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { for (instance, function_coverage) in function_coverage_map { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); let mangled_function_name = tcx.symbol_name(instance).to_string(); - let function_source_hash = function_coverage.source_hash(); + let source_hash = function_coverage.source_hash(); + let is_used = function_coverage.is_used(); let (expressions, counter_regions) = function_coverage.get_expressions_and_counter_regions(); @@ -69,7 +77,7 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { "Every `FunctionCoverage` should have at least one counter" ); - function_data.push((mangled_function_name, function_source_hash, coverage_mapping_buffer)); + function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); } // Encode all filenames referenced by counters/expressions in this module @@ -84,13 +92,14 @@ pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { // Generate the LLVM IR representation of the coverage map and store it in a well-known global let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); - for (mangled_function_name, function_source_hash, coverage_mapping_buffer) in function_data { + for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { save_function_record( cx, mangled_function_name, - function_source_hash, + source_hash, filenames_ref, coverage_mapping_buffer, + is_used, ); } @@ -201,9 +210,10 @@ impl CoverageMapGenerator { fn save_function_record( cx: &CodegenCx<'ll, 'tcx>, mangled_function_name: String, - function_source_hash: u64, + source_hash: u64, filenames_ref: u64, coverage_mapping_buffer: Vec<u8>, + is_used: bool, ) { // Concatenate the encoded coverage mappings let coverage_mapping_size = coverage_mapping_buffer.len(); @@ -212,128 +222,120 @@ fn save_function_record( let func_name_hash = coverageinfo::hash_str(&mangled_function_name); let func_name_hash_val = cx.const_u64(func_name_hash); let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); - let func_hash_val = cx.const_u64(function_source_hash); + let source_hash_val = cx.const_u64(source_hash); let filenames_ref_val = cx.const_u64(filenames_ref); let func_record_val = cx.const_struct( &[ func_name_hash_val, coverage_mapping_size_val, - func_hash_val, + source_hash_val, filenames_ref_val, coverage_mapping_val, ], /*packed=*/ true, ); - // At the present time, the coverage map for Rust assumes every instrumented function `is_used`. - // Note that Clang marks functions as "unused" in `CodeGenPGO::emitEmptyCounterMapping`. (See: - // https://github.com/rust-lang/llvm-project/blob/de02a75e398415bad4df27b4547c25b896c8bf3b/clang%2Flib%2FCodeGen%2FCodeGenPGO.cpp#L877-L878 - // for example.) - // - // It's not yet clear if or how this may be applied to Rust in the future, but the `is_used` - // argument is available and handled similarly. - let is_used = true; coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); } /// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for /// the functions that went through codegen; such as public functions and "used" functions /// (functions referenced by other "used" or public items). Any other functions considered unused, -/// or "Unreachable" were still parsed and processed through the MIR stage. +/// or "Unreachable", were still parsed and processed through the MIR stage, but were not +/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but +/// that flag is known to cause other errors, when combined with `-Z instrument-coverage`; and +/// `-Clink-dead-code` will not generate code for unused generic functions.) /// -/// We can find the unreachable functions by the set difference of all MIR `DefId`s (`tcx` query -/// `mir_keys`) minus the codegenned `DefId`s (`tcx` query `collect_and_partition_mono_items`). +/// We can find the unused functions (including generic functions) by the set difference of all MIR +/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query +/// `collect_and_partition_mono_items`). /// /// *HOWEVER* the codegenned `DefId`s are partitioned across multiple `CodegenUnit`s (CGUs), and /// this function is processing a `function_coverage_map` for the functions (`Instance`/`DefId`) -/// allocated to only one of those CGUs. We must NOT inject any "Unreachable" functions's -/// `CodeRegion`s more than once, so we have to pick which CGU's `function_coverage_map` to add -/// each "Unreachable" function to. -/// -/// Some constraints: -/// -/// 1. The file name of an "Unreachable" function must match the file name of the existing -/// codegenned (covered) function to which the unreachable code regions will be added. -/// 2. The function to which the unreachable code regions will be added must not be a genaric -/// function (must not have type parameters) because the coverage tools will get confused -/// if the codegenned function has more than one instantiation and additional `CodeRegion`s -/// attached to only one of those instantiations. -fn add_unreachable_coverage<'tcx>( - tcx: TyCtxt<'tcx>, - function_coverage_map: &mut FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>, -) { +/// allocated to only one of those CGUs. We must NOT inject any unused functions's `CodeRegion`s +/// more than once, so we have to pick a CGUs `function_coverage_map` into which the unused +/// function will be inserted. +fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + let tcx = cx.tcx; + // FIXME(#79622): Can this solution be simplified and/or improved? Are there other sources // of compiler state data that might help (or better sources that could be exposed, but // aren't yet)? - // Note: If the crate *only* defines generic functions, there are no codegenerated non-generic - // functions to add any unreachable code to. In this case, the unreachable code regions will - // have no coverage, instead of having coverage with zero executions. - // - // This is probably still an improvement over Clang, which does not generate any coverage - // for uninstantiated template functions. - - let has_non_generic_def_ids = - function_coverage_map.keys().any(|instance| instance.def.attrs(tcx).len() == 0); - - if !has_non_generic_def_ids { - // There are no non-generic functions to add unreachable `CodeRegion`s to - return; - } + let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); - let all_def_ids: DefIdSet = - tcx.mir_keys(LOCAL_CRATE).iter().map(|local_def_id| local_def_id.to_def_id()).collect(); + let all_def_ids: DefIdSet = tcx + .mir_keys(LOCAL_CRATE) + .iter() + .filter_map(|local_def_id| { + let def_id = local_def_id.to_def_id(); + if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) { + return None; + } + Some(local_def_id.to_def_id()) + }) + .collect(); - let (codegenned_def_ids, _) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); + let codegenned_def_ids = tcx.codegened_and_inlined_items(LOCAL_CRATE); - let mut unreachable_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default(); + let mut unused_def_ids_by_file: FxHashMap<Symbol, Vec<DefId>> = FxHashMap::default(); for &non_codegenned_def_id in all_def_ids.difference(codegenned_def_ids) { - // Make sure the non-codegenned (unreachable) function has a file_name + // Make sure the non-codegenned (unused) function has a file_name if let Some(non_codegenned_file_name) = tcx.covered_file_name(non_codegenned_def_id) { - let def_ids = unreachable_def_ids_by_file - .entry(*non_codegenned_file_name) - .or_insert_with(Vec::new); + let def_ids = + unused_def_ids_by_file.entry(*non_codegenned_file_name).or_insert_with(Vec::new); def_ids.push(non_codegenned_def_id); } } - if unreachable_def_ids_by_file.is_empty() { - // There are no unreachable functions with file names to add (in any CGU) + if unused_def_ids_by_file.is_empty() { + // There are no unused functions with file names to add (in any CGU) return; } - // Since there may be multiple `CodegenUnit`s, some codegenned_def_ids may be codegenned in a - // different CGU, and will be added to the function_coverage_map for each CGU. Determine which - // function_coverage_map has the responsibility for publishing unreachable coverage - // based on file name: + // Each `CodegenUnit` (CGU) has its own function_coverage_map, and generates a specific binary + // with its own coverage map. + // + // Each covered function `Instance` can be included in only one coverage map, produced from a + // specific function_coverage_map, from a specific CGU. // - // For each covered file name, sort ONLY the non-generic codegenned_def_ids, and if - // covered_def_ids.contains(the first def_id) for a given file_name, add the unreachable code - // region in this function_coverage_map. Otherwise, ignore it and assume another CGU's - // function_coverage_map will be adding it (because it will be first for one, and only one, - // of them). + // Since unused functions did not generate code, they are not associated with any CGU yet. + // + // To avoid injecting the unused functions in multiple coverage maps (for multiple CGUs) + // determine which function_coverage_map has the responsibility for publishing unreachable + // coverage, based on file name: For each unused function, find the CGU that generates the + // first function (based on sorted `DefId`) from the same file. + // + // Add a new `FunctionCoverage` to the `function_coverage_map`, with unreachable code regions + // for each region in it's MIR. + + // Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them. let mut sorted_codegenned_def_ids: Vec<DefId> = codegenned_def_ids.iter().map(|def_id| *def_id).collect(); sorted_codegenned_def_ids.sort_unstable(); let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default(); for &def_id in sorted_codegenned_def_ids.iter() { - // Only consider non-generic functions, to potentially add unreachable code regions - if tcx.generics_of(def_id).count() == 0 { - if let Some(covered_file_name) = tcx.covered_file_name(def_id) { - // Only add files known to have unreachable functions - if unreachable_def_ids_by_file.contains_key(covered_file_name) { - first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id); - } + if let Some(covered_file_name) = tcx.covered_file_name(def_id) { + // Only add files known to have unused functions + if unused_def_ids_by_file.contains_key(covered_file_name) { + first_covered_def_id_by_file.entry(*covered_file_name).or_insert(def_id); } } } // Get the set of def_ids with coverage regions, known by *this* CoverageContext. - let cgu_covered_def_ids: DefIdSet = - function_coverage_map.keys().map(|instance| instance.def.def_id()).collect(); + let cgu_covered_def_ids: DefIdSet = match cx.coverage_context() { + Some(ctx) => ctx + .function_coverage_map + .borrow() + .keys() + .map(|&instance| instance.def.def_id()) + .collect(), + None => return, + }; - let mut cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file + let cgu_covered_files: FxHashSet<Symbol> = first_covered_def_id_by_file .iter() .filter_map( |(&file_name, def_id)| { @@ -342,49 +344,13 @@ fn add_unreachable_coverage<'tcx>( ) .collect(); - // Find the first covered, non-generic function (instance) for each cgu_covered_file. Take the - // unreachable code regions for that file, and add them to the function. - // - // There are three `for` loops here, but (a) the lists have already been reduced to the minimum - // required values, the lists are further reduced (by `remove()` calls) when elements are no - // longer needed, and there are several opportunities to branch out of loops early. - for (instance, function_coverage) in function_coverage_map.iter_mut() { - if instance.def.attrs(tcx).len() > 0 { - continue; - } - // The covered function is not generic... - let covered_def_id = instance.def.def_id(); - if let Some(covered_file_name) = tcx.covered_file_name(covered_def_id) { - if !cgu_covered_files.remove(&covered_file_name) { - continue; - } - // The covered function's file is one of the files with unreachable code regions, so - // all of the unreachable code regions for this file will be added to this function. - for def_id in - unreachable_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() - { - // Note, this loop adds an unreachable code regions for each MIR-derived region. - // Alternatively, we could add a single code region for the maximum span of all - // code regions here. - // - // Observed downsides of this approach are: - // - // 1. The coverage results will appear inconsistent compared with the same (or - // similar) code in a function that is reached. - // 2. If the function is unreachable from one crate but reachable when compiling - // another referencing crate (such as a cross-crate reference to a - // generic function or inlined function), actual coverage regions overlaid - // on a single larger code span of `Zero` coverage can appear confusing or - // wrong. Chaning the unreachable coverage from a `code_region` to a - // `gap_region` can help, but still can look odd with `0` line counts for - // lines between executed (> 0) lines (such as for blank lines or comments). - for ®ion in tcx.covered_code_regions(def_id) { - function_coverage.add_unreachable_region(region.clone()); - } - } - if cgu_covered_files.is_empty() { - break; - } + // For each file for which this CGU is responsible for adding unused function coverage, + // get the `def_id`s for each unused function (if any), define a synthetic function with a + // single LLVM coverage counter, and add the function's coverage `CodeRegion`s. to the + // function_coverage_map. + for covered_file_name in cgu_covered_files { + for def_id in unused_def_ids_by_file.remove(&covered_file_name).into_iter().flatten() { + cx.define_unused_fn(def_id); } } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index e47b8fde40f..afc2bdbfd52 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,5 +1,6 @@ use crate::llvm; +use crate::abi::{Abi, FnAbi}; use crate::builder::Builder; use crate::common::CodegenCx; @@ -7,33 +8,47 @@ use libc::c_uint; use llvm::coverageinfo::CounterMappingRegion; use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; use rustc_codegen_ssa::traits::{ - BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, MiscMethods, StaticMethods, + BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, + MiscMethods, StaticMethods, }; use rustc_data_structures::fx::FxHashMap; +use rustc_hir as hir; +use rustc_hir::def_id::DefId; use rustc_llvm::RustString; +use rustc_middle::bug; use rustc_middle::mir::coverage::{ CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, }; +use rustc_middle::ty; +use rustc_middle::ty::layout::FnAbiExt; +use rustc_middle::ty::subst::InternalSubsts; use rustc_middle::ty::Instance; use std::cell::RefCell; use std::ffi::CString; +use std::iter; use tracing::debug; pub mod mapgen; +const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; + const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. -pub struct CrateCoverageContext<'tcx> { +pub struct CrateCoverageContext<'ll, 'tcx> { // Coverage data for each instrumented function identified by DefId. pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, + pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } -impl<'tcx> CrateCoverageContext<'tcx> { +impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { pub fn new() -> Self { - Self { function_coverage_map: Default::default() } + Self { + function_coverage_map: Default::default(), + pgo_func_name_var_map: Default::default(), + } } pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { @@ -41,23 +56,47 @@ impl<'tcx> CrateCoverageContext<'tcx> { } } -impl CoverageInfoMethods for CodegenCx<'ll, 'tcx> { +impl CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn coverageinfo_finalize(&self) { mapgen::finalize(self) } -} -impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { - /// Calls llvm::createPGOFuncNameVar() with the given function instance's mangled function name. - /// The LLVM API returns an llvm::GlobalVariable containing the function name, with the specific - /// variable name and linkage required by LLVM InstrProf source-based coverage instrumentation. - fn create_pgo_func_name_var(&self, instance: Instance<'tcx>) -> Self::Value { - let llfn = self.cx.get_fn(instance); - let mangled_fn_name = CString::new(self.tcx.symbol_name(instance).name) - .expect("error converting function name to C string"); - unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } + fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { + if let Some(coverage_context) = self.coverage_context() { + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map + .entry(instance) + .or_insert_with(|| create_pgo_func_name_var(self, instance)) + } else { + bug!("Could not get the `coverage_context`"); + } + } + + /// Functions with MIR-based coverage are normally codegenned _only_ if + /// called. LLVM coverage tools typically expect every function to be + /// defined (even if unused), with at least one call to LLVM intrinsic + /// `instrprof.increment`. + /// + /// Codegen a small function that will never be called, with one counter + /// that will never be incremented. + /// + /// For used/called functions, the coverageinfo was already added to the + /// `function_coverage_map` (keyed by function `Instance`) during codegen. + /// But in this case, since the unused function was _not_ previously + /// codegenned, collect the coverage `CodeRegion`s from the MIR and add + /// them. The first `CodeRegion` is used to add a single counter, with the + /// same counter ID used in the injected `instrprof.increment` intrinsic + /// call. Since the function is never called, all other `CodeRegion`s can be + /// added as `unreachable_region`s. + fn define_unused_fn(&self, def_id: DefId) { + let instance = declare_unused_fn(self, &def_id); + codegen_unused_fn_and_counter(self, instance); + add_unused_function_coverage(self, instance, def_id); } +} +impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { fn set_function_source_hash( &mut self, instance: Instance<'tcx>, @@ -145,6 +184,100 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } +fn declare_unused_fn(cx: &CodegenCx<'ll, 'tcx>, def_id: &DefId) -> Instance<'tcx> { + let tcx = cx.tcx; + + let instance = Instance::new( + *def_id, + InternalSubsts::for_item(tcx, *def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ); + + let llfn = cx.declare_fn( + &tcx.symbol_name(instance).name, + &FnAbi::of_fn_ptr( + cx, + ty::Binder::dummy(tcx.mk_fn_sig( + iter::once(tcx.mk_unit()), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + &[], + ), + ); + + llvm::set_linkage(llfn, llvm::Linkage::WeakAnyLinkage); + llvm::set_visibility(llfn, llvm::Visibility::Hidden); + + assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); + + instance +} + +fn codegen_unused_fn_and_counter(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) { + let llfn = cx.get_fn(instance); + let mut bx = Builder::new_block(cx, llfn, "unused_function"); + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(0); + let num_counters = bx.const_u32(1); + let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, + index={:?}) for unused function: {:?}", + fn_name, hash, num_counters, index, instance + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + bx.ret_void(); +} + +fn add_unused_function_coverage( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + def_id: DefId, +) { + let tcx = cx.tcx; + + let mut function_coverage = FunctionCoverage::unused(tcx, instance); + for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { + if index == 0 { + // Insert at least one real counter so the LLVM CoverageMappingReader will find expected + // definitions. + function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); + } else { + function_coverage.add_unreachable_region(code_region.clone()); + } + } + + if let Some(coverage_context) = cx.coverage_context() { + coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); + } else { + bug!("Could not get the `coverage_context`"); + } +} + +/// Calls llvm::createPGOFuncNameVar() with the given function instance's +/// mangled function name. The LLVM API returns an llvm::GlobalVariable +/// containing the function name, with the specific variable name and linkage +/// required by LLVM InstrProf source-based coverage instrumentation. Use +/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +/// `Instance`. +fn create_pgo_func_name_var( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, +) -> &'ll llvm::Value { + let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) + .expect("error converting function name to C string"); + let llfn = cx.get_fn(instance); + unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } +} + pub(crate) fn write_filenames_section_to_buffer<'a>( filenames: impl IntoIterator<Item = &'a CString>, buffer: &RustString, @@ -177,6 +310,7 @@ pub(crate) fn write_mapping_to_buffer( ); } } + pub(crate) fn hash_str(strval: &str) -> u64 { let strval = CString::new(strval).expect("null error converting hashable str to C string"); unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md new file mode 100644 index 00000000000..f983d092039 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md @@ -0,0 +1,180 @@ +# Debug Info Module + +This module serves the purpose of generating debug symbols. We use LLVM's +[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html) +features for generating the debug information. The general principle is +this: + +Given the right metadata in the LLVM IR, the LLVM code generator is able to +create DWARF debug symbols for the given code. The +[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured +much like DWARF *debugging information entries* (DIE), representing type +information such as datatype layout, function signatures, block layout, +variable location and scope information, etc. It is the purpose of this +module to generate correct metadata and insert it into the LLVM IR. + +As the exact format of metadata trees may change between different LLVM +versions, we now use LLVM +[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html) +to create metadata where possible. This will hopefully ease the adaption of +this module to future LLVM versions. + +The public API of the module is a set of functions that will insert the +correct metadata into the LLVM IR when called with the right parameters. +The module is thus driven from an outside client with functions like +`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`. + +Internally the module will try to reuse already created metadata by +utilizing a cache. The way to get a shared metadata node when needed is +thus to just call the corresponding function in this module: + + let file_metadata = file_metadata(cx, file); + +The function will take care of probing the cache for an existing node for +that exact file path. + +All private state used by the module is stored within either the +CrateDebugContext struct (owned by the CodegenCx) or the +FunctionDebugContext (owned by the FunctionCx). + +This file consists of three conceptual sections: +1. The public interface of the module +2. Module-internal metadata creation functions +3. Minor utility functions + + +## Recursive Types + +Some kinds of types, such as structs and enums can be recursive. That means +that the type definition of some type X refers to some other type which in +turn (transitively) refers to X. This introduces cycles into the type +referral graph. A naive algorithm doing an on-demand, depth-first traversal +of this graph when describing types, can get trapped in an endless loop +when it reaches such a cycle. + +For example, the following simple type for a singly-linked list... + +``` +struct List { + value: i32, + tail: Option<Box<List>>, +} +``` + +will generate the following callstack with a naive DFS algorithm: + +``` +describe(t = List) + describe(t = i32) + describe(t = Option<Box<List>>) + describe(t = Box<List>) + describe(t = List) // at the beginning again... + ... +``` + +To break cycles like these, we use "forward declarations". That is, when +the algorithm encounters a possibly recursive type (any struct or enum), it +immediately creates a type description node and inserts it into the cache +*before* describing the members of the type. This type description is just +a stub (as type members are not described and added to it yet) but it +allows the algorithm to already refer to the type. After the stub is +inserted into the cache, the algorithm continues as before. If it now +encounters a recursive reference, it will hit the cache and does not try to +describe the type anew. + +This behavior is encapsulated in the 'RecursiveTypeDescription' enum, +which represents a kind of continuation, storing all state needed to +continue traversal at the type members after the type has been registered +with the cache. (This implementation approach might be a tad over- +engineered and may change in the future) + + +## Source Locations and Line Information + +In addition to data type descriptions the debugging information must also +allow to map machine code locations back to source code locations in order +to be useful. This functionality is also handled in this module. The +following functions allow to control source mappings: + ++ `set_source_location()` ++ `clear_source_location()` ++ `start_emitting_source_locations()` + +`set_source_location()` allows to set the current source location. All IR +instructions created after a call to this function will be linked to the +given source location, until another location is specified with +`set_source_location()` or the source location is cleared with +`clear_source_location()`. In the later case, subsequent IR instruction +will not be linked to any source location. As you can see, this is a +stateful API (mimicking the one in LLVM), so be careful with source +locations set by previous calls. It's probably best to not rely on any +specific state being present at a given point in code. + +One topic that deserves some extra attention is *function prologues*. At +the beginning of a function's machine code there are typically a few +instructions for loading argument values into allocas and checking if +there's enough stack space for the function to execute. This *prologue* is +not visible in the source code and LLVM puts a special PROLOGUE END marker +into the line table at the first non-prologue instruction of the function. +In order to find out where the prologue ends, LLVM looks for the first +instruction in the function body that is linked to a source location. So, +when generating prologue instructions we have to make sure that we don't +emit source location information until the 'real' function body begins. For +this reason, source location emission is disabled by default for any new +function being codegened and is only activated after a call to the third +function from the list above, `start_emitting_source_locations()`. This +function should be called right before regularly starting to codegen the +top-level block of the given function. + +There is one exception to the above rule: `llvm.dbg.declare` instruction +must be linked to the source location of the variable being declared. For +function parameters these `llvm.dbg.declare` instructions typically occur +in the middle of the prologue, however, they are ignored by LLVM's prologue +detection. The `create_argument_metadata()` and related functions take care +of linking the `llvm.dbg.declare` instructions to the correct source +locations even while source location emission is still disabled, so there +is no need to do anything special with source location handling here. + +## Unique Type Identification + +In order for link-time optimization to work properly, LLVM needs a unique +type identifier that tells it across compilation units which types are the +same as others. This type identifier is created by +`TypeMap::get_unique_type_id_of_type()` using the following algorithm: + +1. Primitive types have their name as ID + +2. Structs, enums and traits have a multipart identifier + + 1. The first part is the SVH (strict version hash) of the crate they + were originally defined in + + 2. The second part is the ast::NodeId of the definition in their + original crate + + 3. The final part is a concatenation of the type IDs of their concrete + type arguments if they are generic types. + +3. Tuple-, pointer-, and function types are structurally identified, which + means that they are equivalent if their component types are equivalent + (i.e., `(i32, i32)` is the same regardless in which crate it is used). + +This algorithm also provides a stable ID for types that are defined in one +crate but instantiated from metadata within another crate. We just have to +take care to always map crate and `NodeId`s back to the original crate +context. + +As a side-effect these unique type IDs also help to solve a problem arising +from lifetime parameters. Since lifetime parameters are completely omitted +in debuginfo, more than one `Ty` instance may map to the same debuginfo +type metadata, that is, some struct `Struct<'a>` may have N instantiations +with different concrete substitutions for `'a`, and thus there will be N +`Ty` instances for the type `Struct<'a>` even though it is not generic +otherwise. Unfortunately this means that we cannot use `ty::type_id()` as +cheap identifier for type metadata -- we have done this in the past, but it +led to unnecessary metadata duplication in the best case and LLVM +assertions in the worst. However, the unique type ID as described above +*can* be used as identifier. Since it is comparatively expensive to +construct, though, `ty::type_id()` is still used additionally as an +optimization for cases where the exact same type has been seen before +(which is most of the time). diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs b/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs deleted file mode 100644 index 10dd5906529..00000000000 --- a/compiler/rustc_codegen_llvm/src/debuginfo/doc.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! # Debug Info Module -//! -//! This module serves the purpose of generating debug symbols. We use LLVM's -//! [source level debugging](https://llvm.org/docs/SourceLevelDebugging.html) -//! features for generating the debug information. The general principle is -//! this: -//! -//! Given the right metadata in the LLVM IR, the LLVM code generator is able to -//! create DWARF debug symbols for the given code. The -//! [metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured -//! much like DWARF *debugging information entries* (DIE), representing type -//! information such as datatype layout, function signatures, block layout, -//! variable location and scope information, etc. It is the purpose of this -//! module to generate correct metadata and insert it into the LLVM IR. -//! -//! As the exact format of metadata trees may change between different LLVM -//! versions, we now use LLVM -//! [DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html) -//! to create metadata where possible. This will hopefully ease the adaption of -//! this module to future LLVM versions. -//! -//! The public API of the module is a set of functions that will insert the -//! correct metadata into the LLVM IR when called with the right parameters. -//! The module is thus driven from an outside client with functions like -//! `debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`. -//! -//! Internally the module will try to reuse already created metadata by -//! utilizing a cache. The way to get a shared metadata node when needed is -//! thus to just call the corresponding function in this module: -//! -//! let file_metadata = file_metadata(cx, file); -//! -//! The function will take care of probing the cache for an existing node for -//! that exact file path. -//! -//! All private state used by the module is stored within either the -//! CrateDebugContext struct (owned by the CodegenCx) or the -//! FunctionDebugContext (owned by the FunctionCx). -//! -//! This file consists of three conceptual sections: -//! 1. The public interface of the module -//! 2. Module-internal metadata creation functions -//! 3. Minor utility functions -//! -//! -//! ## Recursive Types -//! -//! Some kinds of types, such as structs and enums can be recursive. That means -//! that the type definition of some type X refers to some other type which in -//! turn (transitively) refers to X. This introduces cycles into the type -//! referral graph. A naive algorithm doing an on-demand, depth-first traversal -//! of this graph when describing types, can get trapped in an endless loop -//! when it reaches such a cycle. -//! -//! For example, the following simple type for a singly-linked list... -//! -//! ``` -//! struct List { -//! value: i32, -//! tail: Option<Box<List>>, -//! } -//! ``` -//! -//! will generate the following callstack with a naive DFS algorithm: -//! -//! ``` -//! describe(t = List) -//! describe(t = i32) -//! describe(t = Option<Box<List>>) -//! describe(t = Box<List>) -//! describe(t = List) // at the beginning again... -//! ... -//! ``` -//! -//! To break cycles like these, we use "forward declarations". That is, when -//! the algorithm encounters a possibly recursive type (any struct or enum), it -//! immediately creates a type description node and inserts it into the cache -//! *before* describing the members of the type. This type description is just -//! a stub (as type members are not described and added to it yet) but it -//! allows the algorithm to already refer to the type. After the stub is -//! inserted into the cache, the algorithm continues as before. If it now -//! encounters a recursive reference, it will hit the cache and does not try to -//! describe the type anew. -//! -//! This behavior is encapsulated in the 'RecursiveTypeDescription' enum, -//! which represents a kind of continuation, storing all state needed to -//! continue traversal at the type members after the type has been registered -//! with the cache. (This implementation approach might be a tad over- -//! engineered and may change in the future) -//! -//! -//! ## Source Locations and Line Information -//! -//! In addition to data type descriptions the debugging information must also -//! allow to map machine code locations back to source code locations in order -//! to be useful. This functionality is also handled in this module. The -//! following functions allow to control source mappings: -//! -//! + set_source_location() -//! + clear_source_location() -//! + start_emitting_source_locations() -//! -//! `set_source_location()` allows to set the current source location. All IR -//! instructions created after a call to this function will be linked to the -//! given source location, until another location is specified with -//! `set_source_location()` or the source location is cleared with -//! `clear_source_location()`. In the later case, subsequent IR instruction -//! will not be linked to any source location. As you can see, this is a -//! stateful API (mimicking the one in LLVM), so be careful with source -//! locations set by previous calls. It's probably best to not rely on any -//! specific state being present at a given point in code. -//! -//! One topic that deserves some extra attention is *function prologues*. At -//! the beginning of a function's machine code there are typically a few -//! instructions for loading argument values into allocas and checking if -//! there's enough stack space for the function to execute. This *prologue* is -//! not visible in the source code and LLVM puts a special PROLOGUE END marker -//! into the line table at the first non-prologue instruction of the function. -//! In order to find out where the prologue ends, LLVM looks for the first -//! instruction in the function body that is linked to a source location. So, -//! when generating prologue instructions we have to make sure that we don't -//! emit source location information until the 'real' function body begins. For -//! this reason, source location emission is disabled by default for any new -//! function being codegened and is only activated after a call to the third -//! function from the list above, `start_emitting_source_locations()`. This -//! function should be called right before regularly starting to codegen the -//! top-level block of the given function. -//! -//! There is one exception to the above rule: `llvm.dbg.declare` instruction -//! must be linked to the source location of the variable being declared. For -//! function parameters these `llvm.dbg.declare` instructions typically occur -//! in the middle of the prologue, however, they are ignored by LLVM's prologue -//! detection. The `create_argument_metadata()` and related functions take care -//! of linking the `llvm.dbg.declare` instructions to the correct source -//! locations even while source location emission is still disabled, so there -//! is no need to do anything special with source location handling here. -//! -//! ## Unique Type Identification -//! -//! In order for link-time optimization to work properly, LLVM needs a unique -//! type identifier that tells it across compilation units which types are the -//! same as others. This type identifier is created by -//! `TypeMap::get_unique_type_id_of_type()` using the following algorithm: -//! -//! (1) Primitive types have their name as ID -//! (2) Structs, enums and traits have a multipart identifier -//! -//! (1) The first part is the SVH (strict version hash) of the crate they -//! were originally defined in -//! -//! (2) The second part is the ast::NodeId of the definition in their -//! original crate -//! -//! (3) The final part is a concatenation of the type IDs of their concrete -//! type arguments if they are generic types. -//! -//! (3) Tuple-, pointer and function types are structurally identified, which -//! means that they are equivalent if their component types are equivalent -//! (i.e., (i32, i32) is the same regardless in which crate it is used). -//! -//! This algorithm also provides a stable ID for types that are defined in one -//! crate but instantiated from metadata within another crate. We just have to -//! take care to always map crate and `NodeId`s back to the original crate -//! context. -//! -//! As a side-effect these unique type IDs also help to solve a problem arising -//! from lifetime parameters. Since lifetime parameters are completely omitted -//! in debuginfo, more than one `Ty` instance may map to the same debuginfo -//! type metadata, that is, some struct `Struct<'a>` may have N instantiations -//! with different concrete substitutions for `'a`, and thus there will be N -//! `Ty` instances for the type `Struct<'a>` even though it is not generic -//! otherwise. Unfortunately this means that we cannot use `ty::type_id()` as -//! cheap identifier for type metadata -- we have done this in the past, but it -//! led to unnecessary metadata duplication in the best case and LLVM -//! assertions in the worst. However, the unique type ID as described above -//! *can* be used as identifier. Since it is comparatively expensive to -//! construct, though, `ty::type_id()` is still used additionally as an -//! optimization for cases where the exact same type has been seen before -//! (which is most of the time). diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index d5b32e58cc3..e6fa852155b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1083,9 +1083,9 @@ pub fn compile_unit_metadata( ); } - // Insert `llvm.ident` metadata on the wasm32 targets since that will + // Insert `llvm.ident` metadata on the wasm targets since that will // get hooked up to the "producer" sections `processed-by` information. - if tcx.sess.opts.target_triple.triple().starts_with("wasm32") { + if tcx.sess.target.is_like_wasm { let name_metadata = llvm::LLVMMDStringInContext( debug_context.llcontext, rustc_producer.as_ptr().cast(), @@ -1962,9 +1962,7 @@ fn prepare_enum_metadata( let discriminant_type_metadata = |discr: Primitive| { let enumerators_metadata: Vec<_> = match enum_type.kind() { - ty::Adt(def, _) => def - .discriminants(tcx) - .zip(&def.variants) + ty::Adt(def, _) => iter::zip(def.discriminants(tcx), &def.variants) .map(|((_, discr), v)| { let name = v.ident.as_str(); let is_unsigned = match discr.ty.kind() { @@ -2336,9 +2334,7 @@ fn compute_type_parameters(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>) -> &'ll DIAr if substs.types().next().is_some() { let generics = cx.tcx.generics_of(def.did); let names = get_parameter_names(cx, generics); - let template_params: Vec<_> = substs - .iter() - .zip(names) + let template_params: Vec<_> = iter::zip(substs, names) .filter_map(|(kind, name)| { if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 440e4d505fc..e157a38aa03 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -1,5 +1,4 @@ -// See doc.rs for documentation. -mod doc; +#![doc = include_str!("doc.md")] use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; @@ -38,6 +37,7 @@ use rustc_target::abi::{LayoutOf, Primitive, Size}; use libc::c_uint; use smallvec::SmallVec; use std::cell::RefCell; +use std::iter; use tracing::debug; mod create_scope_map; @@ -449,9 +449,7 @@ impl DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // Again, only create type information if full debuginfo is enabled let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { let names = get_parameter_names(cx, generics); - substs - .iter() - .zip(names) + iter::zip(substs, names) .filter_map(|(kind, name)| { if let GenericArgKind::Type(ty) = kind.unpack() { let actual_type = diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 668daa52ed2..af366f93b91 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -334,8 +334,11 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.call(expect, &[cond, self.const_bool(expected)], None) } - fn sideeffect(&mut self, unconditional: bool) { - if unconditional || self.tcx.sess.opts.debugging_opts.insert_sideeffect { + fn sideeffect(&mut self) { + // This kind of check would make a ton of sense in the caller, but currently the only + // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM + // codegen backend being used, and so is unable to check the LLVM version. + if unsafe { llvm::LLVMRustVersionMajor() } < 12 { let fnname = self.get_intrinsic(&("llvm.sideeffect")); self.call(fnname, &[], None); } @@ -390,7 +393,6 @@ fn codegen_msvc_try( ) { let llfn = get_rust_try_fn(bx, &mut |mut bx| { bx.set_personality_fn(bx.eh_personality()); - bx.sideeffect(false); let mut normal = bx.build_sibling_block("normal"); let mut catchswitch = bx.build_sibling_block("catchswitch"); @@ -552,9 +554,6 @@ fn codegen_gnu_try( // (%ptr, _) = landingpad // call %catch_func(%data, %ptr) // ret 1 - - bx.sideeffect(false); - let mut then = bx.build_sibling_block("then"); let mut catch = bx.build_sibling_block("catch"); @@ -614,9 +613,6 @@ fn codegen_emcc_try( // %catch_data[1] = %is_rust_panic // call %catch_func(%data, %catch_data) // ret 1 - - bx.sideeffect(false); - let mut then = bx.build_sibling_block("then"); let mut catch = bx.build_sibling_block("catch"); @@ -1632,7 +1628,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, out_elem ); } - macro_rules! arith { + macro_rules! arith_binary { ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { $(if name == sym::$name { match in_elem.kind() { @@ -1648,7 +1644,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, })* } } - arith! { + arith_binary! { simd_add: Uint, Int => add, Float => fadd; simd_sub: Uint, Int => sub, Float => fsub; simd_mul: Uint, Int => mul, Float => fmul; @@ -1663,6 +1659,25 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, simd_fmin: Float => minnum; } + macro_rules! arith_unary { + ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { + $(if name == sym::$name { + match in_elem.kind() { + $($(ty::$p(_))|* => { + return Ok(bx.$call(args[0].immediate())) + })* + _ => {}, + } + require!(false, + "unsupported operation on `{}` with element `{}`", + in_ty, + in_elem) + })* + } + } + arith_unary! { + simd_neg: Int => neg, Float => fneg; + } if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { let lhs = args[0].immediate(); diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index d11c1592f99..5ca4b226c38 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -8,10 +8,12 @@ #![feature(bool_to_option)] #![feature(const_cstr_unchecked)] #![feature(crate_visibility_modifier)] +#![feature(extended_key_value_attributes)] #![feature(extern_types)] #![feature(in_band_lifetimes)] +#![feature(iter_zip)] #![feature(nll)] -#![feature(or_patterns)] +#![cfg_attr(bootstrap, feature(or_patterns))] #![recursion_limit = "256"] use back::write::{create_informational_target_machine, create_target_machine}; diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index e6d60044c84..70f78c07c65 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -190,33 +190,6 @@ pub enum RealPredicate { RealPredicateTrue = 15, } -impl RealPredicate { - pub fn from_generic(realpred: rustc_codegen_ssa::common::RealPredicate) -> Self { - match realpred { - rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => { - RealPredicate::RealPredicateFalse - } - rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ, - rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT, - rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE, - rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT, - rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE, - rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE, - rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD, - rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO, - rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ, - rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT, - rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE, - rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT, - rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE, - rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE, - rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => { - RealPredicate::RealPredicateTrue - } - } - } -} - /// LLVMTypeKind #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] @@ -711,7 +684,7 @@ pub mod coverageinfo { } impl CounterMappingRegion { - pub fn code_region( + crate fn code_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -731,7 +704,10 @@ pub mod coverageinfo { } } - pub fn expansion_region( + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + crate fn expansion_region( file_id: u32, expanded_file_id: u32, start_line: u32, @@ -751,7 +727,10 @@ pub mod coverageinfo { } } - pub fn skipped_region( + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + crate fn skipped_region( file_id: u32, start_line: u32, start_col: u32, @@ -770,7 +749,10 @@ pub mod coverageinfo { } } - pub fn gap_region( + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + crate fn gap_region( counter: coverage_map::Counter, file_id: u32, start_line: u32, @@ -1031,6 +1013,7 @@ extern "C" { pub fn LLVMSetSection(Global: &Value, Section: *const c_char); pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); + pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); pub fn LLVMGetAlignment(Global: &Value) -> c_uint; pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); @@ -2326,6 +2309,7 @@ extern "C" { pub fn LLVMRustUnsetComdat(V: &Value); pub fn LLVMRustSetModulePICLevel(M: &Module); pub fn LLVMRustSetModulePIELevel(M: &Module); + pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 544ef38c12c..e80de2bc902 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -98,6 +98,9 @@ unsafe fn configure_llvm(sess: &Session) { // during inlining. Unfortunately these may block other optimizations. add("-preserve-alignment-assumptions-during-inlining=false", false); + // Use non-zero `import-instr-limit` multiplier for cold callsites. + add("-import-cold-multiplier=0.1", false); + for arg in sess_args { add(&(*arg), true); } @@ -218,13 +221,39 @@ pub fn target_cpu(sess: &Session) -> &str { handle_native(name) } -pub fn handle_native_features(sess: &Session) -> Vec<String> { +/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, +/// `--target` and similar). +// FIXME(nagisa): Cache the output of this somehow? Maybe make this a query? We're calling this +// for every function that has `#[target_feature]` on it. The global features won't change between +// the functions; only crates, maybe… +pub fn llvm_global_features(sess: &Session) -> Vec<String> { + // FIXME(nagisa): this should definitely be available more centrally and to other codegen backends. + /// These features control behaviour of rustc rather than llvm. + const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"]; + + // Features that come earlier are overriden by conflicting features later in the string. + // Typically we'll want more explicit settings to override the implicit ones, so: + // + // * Features from -Ctarget-cpu=*; are overriden by [^1] + // * Features implied by --target; are overriden by + // * Features from -Ctarget-feature; are overriden by + // * function specific features. + // + // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly + // through LLVM TargetMachine implementation. + // + // FIXME(nagisa): it isn't clear what's the best interaction between features implied by + // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always + // override anything that's implicit, so e.g. when there's no `--target` flag, features implied + // the host target are overriden by `-Ctarget-cpu=*`. On the other hand, what about when both + // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both + // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence + // should be taken in cases like these. + let mut features = vec![]; + + // -Ctarget-cpu=native match sess.opts.cg.target_cpu { - Some(ref s) => { - if s != "native" { - return vec![]; - } - + Some(ref s) if s == "native" => { let features_string = unsafe { let ptr = llvm::LLVMGetHostCPUFeatures(); let features_string = if !ptr.is_null() { @@ -242,11 +271,31 @@ pub fn handle_native_features(sess: &Session) -> Vec<String> { features_string }; - - features_string.split(",").map(|s| s.to_owned()).collect() + features.extend(features_string.split(",").map(String::from)); } - None => vec![], - } + Some(_) | None => {} + }; + + // Features implied by an implicit or explicit `--target`. + features.extend( + sess.target + .features + .split(',') + .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s))) + .map(String::from), + ); + + // -Ctarget-features + features.extend( + sess.opts + .cg + .target_feature + .split(',') + .filter(|f| !f.is_empty() && !RUSTC_SPECIFIC_FEATURES.iter().any(|s| f.contains(s))) + .map(String::from), + ); + + features } pub fn tune_cpu(sess: &Session) -> Option<&str> { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 992e83d08fc..fc1f364e9c6 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -10,7 +10,9 @@ pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::FnAbiExt; use rustc_middle::ty::{self, Instance, TypeFoldable}; +use rustc_session::config::CrateType; use rustc_target::abi::LayoutOf; +use rustc_target::spec::RelocModel; use tracing::debug; impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { @@ -35,6 +37,9 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); + if self.should_assume_dso_local(linkage, visibility) { + llvm::LLVMRustSetDSOLocal(g, true); + } } self.instances.borrow_mut().insert(instance, g); @@ -79,6 +84,42 @@ impl PreDefineMethods<'tcx> for CodegenCx<'ll, 'tcx> { attributes::from_fn_attrs(self, lldecl, instance); + unsafe { + if self.should_assume_dso_local(linkage, visibility) { + llvm::LLVMRustSetDSOLocal(lldecl, true); + } + } + self.instances.borrow_mut().insert(instance, lldecl); } } + +impl CodegenCx<'ll, 'tcx> { + /// Whether a definition (NB: not declaration!) can be assumed to be local to a group of + /// libraries that form a single DSO or executable. + pub(crate) unsafe fn should_assume_dso_local( + &self, + linkage: Linkage, + visibility: Visibility, + ) -> bool { + if matches!(linkage, Linkage::Internal | Linkage::Private) { + return true; + } + + if visibility != Visibility::Default && linkage != Linkage::ExternalWeak { + return true; + } + + // Static relocation model should force copy relocations everywhere. + if self.tcx.sess.relocation_model() == RelocModel::Static { + return true; + } + + // Symbols from executables can't really be imported any further. + if self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) { + return true; + } + + return false; + } +} |