diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
28 files changed, 1121 insertions, 944 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 9e834b83df4..a5ffe0650a8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -348,45 +348,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Direct(_) => { // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), // and for Scalar ABIs the LLVM type is fully determined by `layout.abi`, - // guarnateeing that we generate ABI-compatible LLVM IR. Things get tricky for - // aggregates... - if matches!(arg.layout.abi, abi::Abi::Aggregate { .. }) { - assert!( - arg.layout.is_sized(), - "`PassMode::Direct` for unsized type: {}", - arg.layout.ty - ); - // This really shouldn't happen, since `immediate_llvm_type` will use - // `layout.fields` to turn this Rust type into an LLVM type. This means all - // sorts of Rust type details leak into the ABI. However wasm sadly *does* - // currently use this mode so we have to allow it -- but we absolutely - // shouldn't let any more targets do that. - // (Also see <https://github.com/rust-lang/rust/issues/115666>.) - assert!( - matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64"), - "`PassMode::Direct` for aggregates only allowed on wasm targets\nProblematic type: {:#?}", - arg.layout, - ); - } + // guaranteeing that we generate ABI-compatible LLVM IR. arg.layout.immediate_llvm_type(cx) } PassMode::Pair(..) => { // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges), // so for ScalarPair we can easily be sure that we are generating ABI-compatible // LLVM IR. - assert!( - matches!(arg.layout.abi, abi::Abi::ScalarPair(..)), - "PassMode::Pair for type {}", - arg.layout.ty - ); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack } => { - // `Indirect` with metadata is only for unsized types, and doesn't work with - // on-stack passing. - assert!(arg.layout.is_unsized() && !on_stack); + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { // Construct the type of a (wide) pointer to `ty`, and pass its two fields. // Any two ABI-compatible unsized types have the same metadata type and // moreover the same metadata value leads to the same dynamic size and @@ -397,13 +370,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); continue; } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { - assert!(arg.layout.is_sized()); - cx.type_ptr() - } + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(), PassMode::Cast { cast, pad_i32 } => { - // `Cast` means "transmute to `CastType`"; that only makes sense for sized types. - assert!(arg.layout.is_sized()); // add padding if *pad_i32 { llargument_tys.push(Reg::i32().llvm_type(cx)); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index ba263296bb4..8655aeec13d 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -2,7 +2,7 @@ use crate::back::write::{ self, bitcode_section_name, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers, }; use crate::errors::{ - DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, + DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro, }; use crate::llvm::{self, build_string}; use crate::{LlvmCodegenBackend, ModuleLlvm}; @@ -19,7 +19,6 @@ use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; -use rustc_session::cgu_reuse_tracker::CguReuse; use rustc_session::config::{self, CrateType, Lto}; use std::ffi::{CStr, CString}; @@ -36,8 +35,12 @@ pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { - CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true, - CrateType::Rlib | CrateType::ProcMacro => false, + CrateType::Executable + | CrateType::Dylib + | CrateType::Staticlib + | CrateType::Cdylib + | CrateType::ProcMacro => true, + CrateType::Rlib => false, } } @@ -87,6 +90,11 @@ fn prepare_lto( diag_handler.emit_err(LtoDylib); return Err(FatalError); } + } else if *crate_type == CrateType::ProcMacro { + if !cgcx.opts.unstable_opts.dylib_lto { + diag_handler.emit_err(LtoProcMacro); + return Err(FatalError); + } } } @@ -575,7 +583,6 @@ fn thin_lto( copy_jobs.push(work_product); info!(" - {}: re-used", module_name); assert!(cgcx.incr_comp_session_dir.is_some()); - cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto); continue; } } diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs new file mode 100644 index 00000000000..36484c3c3fc --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -0,0 +1,103 @@ +use std::{ + ffi::{c_char, CStr}, + marker::PhantomData, + ops::Deref, + ptr::NonNull, +}; + +use rustc_data_structures::small_c_str::SmallCStr; + +use crate::{errors::LlvmError, llvm}; + +/// Responsible for safely creating and disposing llvm::TargetMachine via ffi functions. +/// Not cloneable as there is no clone function for llvm::TargetMachine. +#[repr(transparent)] +pub struct OwnedTargetMachine { + tm_unique: NonNull<llvm::TargetMachine>, + phantom: PhantomData<llvm::TargetMachine>, +} + +impl OwnedTargetMachine { + pub fn new( + triple: &CStr, + cpu: &CStr, + features: &CStr, + abi: &CStr, + model: llvm::CodeModel, + reloc: llvm::RelocModel, + level: llvm::CodeGenOptLevel, + use_soft_fp: bool, + function_sections: bool, + data_sections: bool, + unique_section_names: bool, + trap_unreachable: bool, + singletree: bool, + asm_comments: bool, + emit_stack_size_section: bool, + relax_elf_relocations: bool, + use_init_array: bool, + split_dwarf_file: &CStr, + output_obj_file: &CStr, + debug_info_compression: &CStr, + force_emulated_tls: bool, + args_cstr_buff: &[u8], + ) -> Result<Self, LlvmError<'static>> { + assert!(args_cstr_buff.len() > 0); + assert!( + *args_cstr_buff.last().unwrap() == 0, + "The last character must be a null terminator." + ); + + // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data + let tm_ptr = unsafe { + llvm::LLVMRustCreateTargetMachine( + triple.as_ptr(), + cpu.as_ptr(), + features.as_ptr(), + abi.as_ptr(), + model, + reloc, + level, + use_soft_fp, + function_sections, + data_sections, + unique_section_names, + trap_unreachable, + singletree, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + split_dwarf_file.as_ptr(), + output_obj_file.as_ptr(), + debug_info_compression.as_ptr(), + force_emulated_tls, + args_cstr_buff.as_ptr() as *const c_char, + args_cstr_buff.len(), + ) + }; + + NonNull::new(tm_ptr) + .map(|tm_unique| Self { tm_unique, phantom: PhantomData }) + .ok_or_else(|| LlvmError::CreateTargetMachine { triple: SmallCStr::from(triple) }) + } +} + +impl Deref for OwnedTargetMachine { + type Target = llvm::TargetMachine; + + fn deref(&self) -> &Self::Target { + // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine + unsafe { self.tm_unique.as_ref() } + } +} + +impl Drop for OwnedTargetMachine { + fn drop(&mut self) { + // SAFETY: constructing ensures we have a valid pointer created by llvm::LLVMRustCreateTargetMachine + // OwnedTargetMachine is not copyable so there is no double free or use after free + unsafe { + llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 1f394a7335c..8d335ff1718 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,4 +1,5 @@ use crate::back::lto::ThinBuffer; +use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, }; @@ -98,8 +99,8 @@ pub fn write_output_file<'ll>( } } -pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { - let config = TargetMachineFactoryConfig { split_dwarf_file: None }; +pub fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine { + let config = TargetMachineFactoryConfig { split_dwarf_file: None, output_obj_file: None }; // Can't use query system here quite yet because this function is invoked before the query // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false); @@ -107,7 +108,7 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm: .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise()) } -pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { +pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { tcx.output_filenames(()).split_dwarf_path( tcx.sess.split_debuginfo(), @@ -117,7 +118,11 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll } else { None }; - let config = TargetMachineFactoryConfig { split_dwarf_file }; + + let output_obj_file = + Some(tcx.output_filenames(()).temp_path(OutputType::Object, Some(mod_name))); + let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }; + target_machine_factory( &tcx.sess, tcx.backend_optimization_level(()), @@ -254,39 +259,47 @@ pub fn target_machine_factory( }; let debuginfo_compression = SmallCStr::new(&debuginfo_compression); + let should_prefer_remapped_for_split_debuginfo_paths = + sess.should_prefer_remapped_for_split_debuginfo_paths(); + Arc::new(move |config: TargetMachineFactoryConfig| { - let split_dwarf_file = - path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; - let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); - - let tm = unsafe { - llvm::LLVMRustCreateTargetMachine( - triple.as_ptr(), - cpu.as_ptr(), - features.as_ptr(), - abi.as_ptr(), - code_model, - reloc_model, - opt_level, - use_softfp, - ffunction_sections, - fdata_sections, - funique_section_names, - trap_unreachable, - singlethread, - asm_comments, - emit_stack_size_section, - relax_elf_relocations, - use_init_array, - split_dwarf_file.as_ptr(), - debuginfo_compression.as_ptr(), - force_emulated_tls, - args_cstr_buff.as_ptr() as *const c_char, - args_cstr_buff.len(), - ) + let path_to_cstring_helper = |path: Option<PathBuf>| -> CString { + let path = path.unwrap_or_default(); + let path = if should_prefer_remapped_for_split_debuginfo_paths { + path_mapping.map_prefix(path).0 + } else { + path.into() + }; + CString::new(path.to_str().unwrap()).unwrap() }; - tm.ok_or_else(|| LlvmError::CreateTargetMachine { triple: triple.clone() }) + let split_dwarf_file = path_to_cstring_helper(config.split_dwarf_file); + let output_obj_file = path_to_cstring_helper(config.output_obj_file); + + OwnedTargetMachine::new( + &triple, + &cpu, + &features, + &abi, + code_model, + reloc_model, + opt_level, + use_softfp, + ffunction_sections, + fdata_sections, + funique_section_names, + trap_unreachable, + singlethread, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + &split_dwarf_file, + &output_obj_file, + &debuginfo_compression, + force_emulated_tls, + &args_cstr_buff, + ) }) } @@ -977,11 +990,7 @@ unsafe fn embed_bitcode( // reason (see issue #90326 for historical background). let is_aix = target_is_aix(cgcx); let is_apple = target_is_apple(cgcx); - if is_apple - || is_aix - || cgcx.opts.target_triple.triple().starts_with("wasm") - || cgcx.opts.target_triple.triple().starts_with("asmjs") - { + if is_apple || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") { // We don't need custom section flags, create LLVM globals. let llconst = common::bytes_in_context(llcx, bitcode); let llglobal = llvm::LLVMAddGlobal( diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index ac6d8f84142..7b259055d40 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -3,6 +3,7 @@ use crate::attributes; use crate::common::Funclet; use crate::context::CodegenCx; use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True}; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; @@ -1225,9 +1226,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn do_not_inline(&mut self, llret: &'ll Value) { - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + if llvm_util::get_version() < (17, 0, 2) { + // Work around https://github.com/llvm/llvm-project/issues/66984. + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + } else { + // Cleanup is always the cold path. + let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]); + } } } @@ -1513,8 +1521,13 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llfn: &'ll Value, ) { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; - if self.tcx.sess.is_sanitizer_cfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { - if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) { + if self.tcx.sess.is_sanitizer_cfi_enabled() + && let Some(fn_abi) = fn_abi + && is_indirect_call + { + if let Some(fn_attrs) = fn_attrs + && fn_attrs.no_sanitize.contains(SanitizerSet::CFI) + { return; } @@ -1551,25 +1564,29 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { llfn: &'ll Value, ) -> Option<llvm::OperandBundleDef<'ll>> { let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) }; - let kcfi_bundle = - if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { - if let Some(fn_attrs) = fn_attrs && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) { - return None; - } + let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled() + && let Some(fn_abi) = fn_abi + && is_indirect_call + { + if let Some(fn_attrs) = fn_attrs + && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI) + { + return None; + } - let mut options = TypeIdOptions::empty(); - if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { - options.insert(TypeIdOptions::GENERALIZE_POINTERS); - } - if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { - options.insert(TypeIdOptions::NORMALIZE_INTEGERS); - } + let mut options = TypeIdOptions::empty(); + if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() { + options.insert(TypeIdOptions::GENERALIZE_POINTERS); + } + if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() { + options.insert(TypeIdOptions::NORMALIZE_INTEGERS); + } - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); - Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) - } else { - None - }; + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) + } else { + None + }; kcfi_bundle } } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 5254c3f9c9a..0c9f7f19551 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -46,8 +46,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> llfn } else { let instance_def_id = instance.def_id(); - let llfn = if tcx.sess.target.arch == "x86" && - let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) + let llfn = if tcx.sess.target.arch == "x86" + && let Some(dllimport) = common::get_dllimport(tcx, instance_def_id, sym) { // Fix for https://github.com/rust-lang/rust/issues/104453 // On x86 Windows, LLVM uses 'L' as the prefix for any private @@ -59,9 +59,19 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // To avoid this, we set the Storage Class to "DllImport" so that // LLVM will prefix the name with `__imp_`. Ideally, we'd like the // existing logic below to set the Storage Class, but it has an - // exemption for MinGW for backwards compatability. - let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); - unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } + // exemption for MinGW for backwards compatibility. + let llfn = cx.declare_fn( + &common::i686_decorated_name( + &dllimport, + common::is_mingw_gnu_toolchain(&tcx.sess.target), + true, + ), + fn_abi, + Some(instance), + ); + unsafe { + llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); + } llfn } else { cx.declare_fn(sym, fn_abi, Some(instance)) diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 0b0816c27b6..d1b643f4967 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -203,6 +203,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { unsafe { llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); } (s.to_owned(), g) diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 95af2f8ef4a..307c1264dc1 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -182,10 +182,17 @@ fn check_and_apply_linkage<'ll, 'tcx>( llvm::LLVMSetInitializer(g2, g1); g2 } - } else if cx.tcx.sess.target.arch == "x86" && - let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) + } else if cx.tcx.sess.target.arch == "x86" + && let Some(dllimport) = common::get_dllimport(cx.tcx, def_id, sym) { - cx.declare_global(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), true), llty) + cx.declare_global( + &common::i686_decorated_name( + &dllimport, + common::is_mingw_gnu_toolchain(&cx.tcx.sess.target), + true, + ), + llty, + ) } else { // Generate an external declaration. // FIXME(nagisa): investigate whether it can be changed into define_global @@ -367,15 +374,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { let g = self.get_static(def_id); - // boolean SSA values are i1, but they have to be stored in i8 slots, - // otherwise some LLVM optimization passes don't work as expected - let mut val_llty = self.val_ty(v); - let v = if val_llty == self.type_i1() { - val_llty = self.type_i8(); - llvm::LLVMConstZExt(v, val_llty) - } else { - v - }; + let val_llty = self.val_ty(v); let instance = Instance::mono(self.tcx, def_id); let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 8e8290279ab..242c6aed906 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -26,8 +26,8 @@ use rustc_middle::{bug, span_bug}; use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; -use rustc_span::source_map::Span; use rustc_span::source_map::Spanned; +use rustc_span::Span; use rustc_target::abi::{ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, }; @@ -160,9 +160,9 @@ pub unsafe fn create_module<'ll>( // Ensure the data-layout values hardcoded remain the defaults. if sess.target.is_builtin { + // tm is disposed by its drop impl let tm = crate::back::write::create_informational_target_machine(tcx.sess); - llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm); - llvm::LLVMRustDisposeTargetMachine(tm); + llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm); let llvm_data_layout = llvm::LLVMGetDataLayoutStr(llmod); let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) @@ -368,6 +368,24 @@ pub unsafe fn create_module<'ll>( llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1), ); + // Add module flags specified via -Z llvm_module_flag + for (key, value, behavior) in &sess.opts.unstable_opts.llvm_module_flag { + let key = format!("{key}\0"); + let behavior = match behavior.as_str() { + "error" => llvm::LLVMModFlagBehavior::Error, + "warning" => llvm::LLVMModFlagBehavior::Warning, + "require" => llvm::LLVMModFlagBehavior::Require, + "override" => llvm::LLVMModFlagBehavior::Override, + "append" => llvm::LLVMModFlagBehavior::Append, + "appendunique" => llvm::LLVMModFlagBehavior::AppendUnique, + "max" => llvm::LLVMModFlagBehavior::Max, + "min" => llvm::LLVMModFlagBehavior::Min, + // We already checked this during option parsing + _ => unreachable!(), + }; + llvm::LLVMRustAddModuleFlag(llmod, behavior, key.as_ptr().cast(), *value) + } + llmod } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 7a82d05ce9e..7ad2d03a5ed 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::coverage::{CounterId, MappedExpressionIndex}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] @@ -30,11 +30,8 @@ pub struct Counter { } impl Counter { - /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the - /// `id` is not used. - pub fn zero() -> Self { - Self { kind: CounterKind::Zero, id: 0 } - } + /// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used. + pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. pub fn counter_value_reference(counter_id: CounterId) -> Self { @@ -42,20 +39,16 @@ impl Counter { } /// Constructs a new `Counter` of kind `Expression`. - pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self { - Self { kind: CounterKind::Expression, id: mapped_expression_index.into() } - } - - /// Returns true if the `Counter` kind is `Zero`. - pub fn is_zero(&self) -> bool { - matches!(self.kind, CounterKind::Zero) + pub(crate) fn expression(expression_id: ExpressionId) -> Self { + Self { kind: CounterKind::Expression, id: expression_id.as_u32() } } - /// An explicitly-named function to get the ID value, making it more obvious - /// that the stored value is now 0-based. - pub fn zero_based_id(&self) -> u32 { - debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero"); - self.id + pub(crate) fn from_term(term: CovTerm) -> Self { + match term { + CovTerm::Zero => Self::ZERO, + CovTerm::Counter(id) => Self::counter_value_reference(id), + CovTerm::Expression(id) => Self::expression(id), + } } } @@ -80,12 +73,6 @@ pub struct CounterExpression { pub rhs: Counter, } -impl CounterExpression { - pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { - Self { kind, lhs, rhs } - } -} - /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. /// /// Must match the layout of `LLVMRustCounterMappingRegionKind`. @@ -172,7 +159,7 @@ impl CounterMappingRegion { ) -> Self { Self { counter, - false_counter: Counter::zero(), + false_counter: Counter::ZERO, file_id, expanded_file_id: 0, start_line, @@ -220,8 +207,8 @@ impl CounterMappingRegion { end_col: u32, ) -> Self { Self { - counter: Counter::zero(), - false_counter: Counter::zero(), + counter: Counter::ZERO, + false_counter: Counter::ZERO, file_id, expanded_file_id, start_line, @@ -243,8 +230,8 @@ impl CounterMappingRegion { end_col: u32, ) -> Self { Self { - counter: Counter::zero(), - false_counter: Counter::zero(), + counter: Counter::ZERO, + false_counter: Counter::ZERO, file_id, expanded_file_id: 0, start_line, @@ -268,7 +255,7 @@ impl CounterMappingRegion { ) -> Self { Self { counter, - false_counter: Counter::zero(), + false_counter: Counter::ZERO, file_id, expanded_file_id: 0, start_line, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs index f1e68af25d4..cd67fafb8e4 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -1,311 +1,270 @@ use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind}; -use rustc_index::{IndexSlice, IndexVec}; -use rustc_middle::bug; +use rustc_data_structures::captures::Captures; +use rustc_data_structures::fx::FxIndexSet; +use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{ - CodeRegion, CounterId, ExpressionId, MappedExpressionIndex, Op, Operand, + CodeRegion, CounterId, CovTerm, Expression, ExpressionId, FunctionCoverageInfo, Mapping, Op, }; use rustc_middle::ty::Instance; -use rustc_middle::ty::TyCtxt; - -#[derive(Clone, Debug, PartialEq)] -pub struct Expression { - lhs: Operand, - op: Op, - rhs: Operand, - region: Option<CodeRegion>, -} +use rustc_span::Symbol; -/// Collects all of the coverage regions associated with (a) injected counters, (b) counter -/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), -/// for a given Function. This struct also stores the `function_source_hash`, -/// computed during instrumentation, and forwarded with counters. -/// -/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap -/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter -/// or expression), but the line or lines in the gap region are not executable (such as lines with -/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count -/// for a gap area is only used as the line execution count if there are no other regions on a -/// line." +/// Holds all of the coverage mapping data associated with a function instance, +/// collected during traversal of `Coverage` statements in the function's MIR. #[derive(Debug)] -pub struct FunctionCoverage<'tcx> { - instance: Instance<'tcx>, - source_hash: u64, +pub struct FunctionCoverageCollector<'tcx> { + /// Coverage info that was attached to this function by the instrumentor. + function_coverage_info: &'tcx FunctionCoverageInfo, is_used: bool, - counters: IndexVec<CounterId, Option<CodeRegion>>, - expressions: IndexVec<ExpressionId, Option<Expression>>, - unreachable_regions: Vec<CodeRegion>, + + /// Tracks which counters have been seen, so that we can identify mappings + /// to counters that were optimized out, and set them to zero. + counters_seen: BitSet<CounterId>, + /// Contains all expression IDs that have been seen in an `ExpressionUsed` + /// coverage statement, plus all expression IDs that aren't directly used + /// by any mappings (and therefore do not have expression-used statements). + /// After MIR traversal is finished, we can conclude that any IDs missing + /// from this set must have had their statements deleted by MIR opts. + expressions_seen: BitSet<ExpressionId>, } -impl<'tcx> FunctionCoverage<'tcx> { +impl<'tcx> FunctionCoverageCollector<'tcx> { /// Creates a new set of coverage data for a used (called) function. - pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, true) + pub fn new( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + ) -> Self { + Self::create(instance, function_coverage_info, true) } /// Creates a new set of coverage data for an unused (never called) function. - pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - Self::create(tcx, instance, false) + pub fn unused( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + ) -> Self { + Self::create(instance, function_coverage_info, false) } - fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { - let coverageinfo = tcx.coverageinfo(instance.def); + fn create( + instance: Instance<'tcx>, + function_coverage_info: &'tcx FunctionCoverageInfo, + is_used: bool, + ) -> Self { + let num_counters = function_coverage_info.num_counters; + let num_expressions = function_coverage_info.expressions.len(); debug!( - "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", - instance, coverageinfo, is_used + "FunctionCoverage::create(instance={instance:?}) has \ + num_counters={num_counters}, num_expressions={num_expressions}, is_used={is_used}" ); + + // Create a filled set of expression IDs, so that expressions not + // directly used by mappings will be treated as "seen". + // (If they end up being unused, LLVM will delete them for us.) + let mut expressions_seen = BitSet::new_filled(num_expressions); + // For each expression ID that is directly used by one or more mappings, + // mark it as not-yet-seen. This indicates that we expect to see a + // corresponding `ExpressionUsed` statement during MIR traversal. + for Mapping { term, .. } in &function_coverage_info.mappings { + if let &CovTerm::Expression(id) = term { + expressions_seen.remove(id); + } + } + Self { - instance, - source_hash: 0, // will be set with the first `add_counter()` + function_coverage_info, is_used, - counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), - expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), - unreachable_regions: Vec::new(), + counters_seen: BitSet::new_empty(num_counters), + expressions_seen, } } - /// Returns true for a used (called) function, and false for an unused function. - pub fn is_used(&self) -> bool { - self.is_used + /// Marks a counter ID as having been seen in a counter-increment statement. + #[instrument(level = "debug", skip(self))] + pub(crate) fn mark_counter_id_seen(&mut self, id: CounterId) { + self.counters_seen.insert(id); } - /// Sets the function source hash value. If called multiple times for the same function, all - /// calls should have the same hash value. - pub fn set_function_source_hash(&mut self, source_hash: u64) { - if self.source_hash == 0 { - self.source_hash = source_hash; - } else { - debug_assert_eq!(source_hash, self.source_hash); - } + /// Marks an expression ID as having been seen in an expression-used statement. + #[instrument(level = "debug", skip(self))] + pub(crate) fn mark_expression_id_seen(&mut self, id: ExpressionId) { + self.expressions_seen.insert(id); } - /// Adds a code region to be counted by an injected counter intrinsic. - pub fn add_counter(&mut self, id: CounterId, region: CodeRegion) { - if let Some(previous_region) = self.counters[id].replace(region.clone()) { - assert_eq!(previous_region, region, "add_counter: code region for id changed"); + /// Identify expressions that will always have a value of zero, and note + /// their IDs in [`ZeroExpressions`]. Mappings that refer to a zero expression + /// can instead become mappings to a constant zero value. + /// + /// This method mainly exists to preserve the simplifications that were + /// already being performed by the Rust-side expression renumbering, so that + /// the resulting coverage mappings don't get worse. + fn identify_zero_expressions(&self) -> ZeroExpressions { + // The set of expressions that either were optimized out entirely, or + // have zero as both of their operands, and will therefore always have + // a value of zero. Other expressions that refer to these as operands + // can have those operands replaced with `CovTerm::Zero`. + let mut zero_expressions = ZeroExpressions::default(); + + // Simplify a copy of each expression based on lower-numbered expressions, + // and then update the set of always-zero expressions if necessary. + // (By construction, expressions can only refer to other expressions + // that have lower IDs, so one pass is sufficient.) + for (id, expression) in self.function_coverage_info.expressions.iter_enumerated() { + if !self.expressions_seen.contains(id) { + // If an expression was not seen, it must have been optimized away, + // so any operand that refers to it can be replaced with zero. + zero_expressions.insert(id); + continue; + } + + // We don't need to simplify the actual expression data in the + // expressions list; we can just simplify a temporary copy and then + // use that to update the set of always-zero expressions. + let Expression { mut lhs, op, mut rhs } = *expression; + + // If an expression has an operand that is also an expression, the + // operand's ID must be strictly lower. This is what lets us find + // all zero expressions in one pass. + let assert_operand_expression_is_lower = |operand_id: ExpressionId| { + assert!( + operand_id < id, + "Operand {operand_id:?} should be less than {id:?} in {expression:?}", + ) + }; + + // If an operand refers to a counter or expression that is always + // zero, then that operand can be replaced with `CovTerm::Zero`. + let maybe_set_operand_to_zero = |operand: &mut CovTerm| { + if let CovTerm::Expression(id) = *operand { + assert_operand_expression_is_lower(id); + } + + if is_zero_term(&self.counters_seen, &zero_expressions, *operand) { + *operand = CovTerm::Zero; + } + }; + maybe_set_operand_to_zero(&mut lhs); + maybe_set_operand_to_zero(&mut rhs); + + // Coverage counter values cannot be negative, so if an expression + // involves subtraction from zero, assume that its RHS must also be zero. + // (Do this after simplifications that could set the LHS to zero.) + if lhs == CovTerm::Zero && op == Op::Subtract { + rhs = CovTerm::Zero; + } + + // After the above simplifications, if both operands are zero, then + // we know that this expression is always zero too. + if lhs == CovTerm::Zero && rhs == CovTerm::Zero { + zero_expressions.insert(id); + } } + + zero_expressions } - /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other - /// expressions. These are tracked as separate variants of `Operand`, so there is no ambiguity - /// between operands that are counter IDs and operands that are expression IDs. - pub fn add_counter_expression( - &mut self, - expression_id: ExpressionId, - lhs: Operand, - op: Op, - rhs: Operand, - region: Option<CodeRegion>, - ) { - debug!( - "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", - expression_id, lhs, op, rhs, region - ); - debug_assert!( - expression_id.as_usize() < self.expressions.len(), - "expression_id {} is out of range for expressions.len() = {} - for {:?}", - expression_id.as_usize(), - self.expressions.len(), - self, - ); - if let Some(previous_expression) = self.expressions[expression_id].replace(Expression { - lhs, - op, - rhs, - region: region.clone(), - }) { - assert_eq!( - previous_expression, - Expression { lhs, op, rhs, region }, - "add_counter_expression: expression for id changed" - ); - } + pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> { + let zero_expressions = self.identify_zero_expressions(); + let FunctionCoverageCollector { function_coverage_info, is_used, counters_seen, .. } = self; + + FunctionCoverage { function_coverage_info, is_used, counters_seen, zero_expressions } } +} + +pub(crate) struct FunctionCoverage<'tcx> { + function_coverage_info: &'tcx FunctionCoverageInfo, + is_used: bool, - /// Add a region that will be marked as "unreachable", with a constant "zero counter". - pub fn add_unreachable_region(&mut self, region: CodeRegion) { - self.unreachable_regions.push(region) + counters_seen: BitSet<CounterId>, + zero_expressions: ZeroExpressions, +} + +impl<'tcx> FunctionCoverage<'tcx> { + /// Returns true for a used (called) function, and false for an unused function. + pub(crate) fn is_used(&self) -> bool { + self.is_used } /// Return the source hash, generated from the HIR node structure, and used to indicate whether /// or not the source code structure changed between different compilations. pub fn source_hash(&self) -> u64 { - self.source_hash + if self.is_used { self.function_coverage_info.function_source_hash } else { 0 } } - /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their - /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create - /// `CounterMappingRegion`s. - pub fn get_expressions_and_counter_regions( - &self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { - assert!( - self.source_hash != 0 || !self.is_used, - "No counters provided the source_hash for used function: {:?}", - self.instance - ); + /// Returns an iterator over all filenames used by this function's mappings. + pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> { + self.function_coverage_info.mappings.iter().map(|mapping| mapping.code_region.file_name) + } - let counter_regions = self.counter_regions(); - let (counter_expressions, expression_regions) = self.expressions_with_regions(); - let unreachable_regions = self.unreachable_regions(); + /// Convert this function's coverage expression data into a form that can be + /// passed through FFI to LLVM. + pub(crate) fn counter_expressions( + &self, + ) -> impl Iterator<Item = CounterExpression> + ExactSizeIterator + Captures<'_> { + // We know that LLVM will optimize out any unused expressions before + // producing the final coverage map, so there's no need to do the same + // thing on the Rust side unless we're confident we can do much better. + // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.) - let counter_regions = - counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); - (counter_expressions, counter_regions) + self.function_coverage_info.expressions.iter().map(move |&Expression { lhs, op, rhs }| { + CounterExpression { + lhs: self.counter_for_term(lhs), + kind: match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + rhs: self.counter_for_term(rhs), + } + }) } - fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.counters.iter_enumerated().filter_map(|(index, entry)| { - // Option::map() will return None to filter out missing counters. This may happen - // if, for example, a MIR-instrumented counter is removed during an optimization. - entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) + /// Converts this function's coverage mappings into an intermediate form + /// that will be used by `mapgen` when preparing for FFI. + pub(crate) fn counter_regions( + &self, + ) -> impl Iterator<Item = (Counter, &CodeRegion)> + ExactSizeIterator { + self.function_coverage_info.mappings.iter().map(move |mapping| { + let &Mapping { term, ref code_region } = mapping; + let counter = self.counter_for_term(term); + (counter, code_region) }) } - fn expressions_with_regions( - &self, - ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { - let mut counter_expressions = Vec::with_capacity(self.expressions.len()); - let mut expression_regions = Vec::with_capacity(self.expressions.len()); - let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); - - // This closure converts any `Expression` operand (`lhs` or `rhs` of the `Op::Add` or - // `Op::Subtract` operation) into its native `llvm::coverage::Counter::CounterKind` type - // and value. - // - // Expressions will be returned from this function in a sequential vector (array) of - // `CounterExpression`, so the expression IDs must be mapped from their original, - // potentially sparse set of indexes. - // - // An `Expression` as an operand will have already been encountered as an `Expression` with - // operands, so its new_index will already have been generated (as a 1-up index value). - // (If an `Expression` as an operand does not have a corresponding new_index, it was - // probably optimized out, after the expression was injected into the MIR, so it will - // get a `CounterKind::Zero` instead.) - // - // In other words, an `Expression`s at any given index can include other expressions as - // operands, but expression operands can only come from the subset of expressions having - // `expression_index`s lower than the referencing `Expression`. Therefore, it is - // reasonable to look up the new index of an expression operand while the `new_indexes` - // vector is only complete up to the current `ExpressionIndex`. - type NewIndexes = IndexSlice<ExpressionId, Option<MappedExpressionIndex>>; - let id_to_counter = |new_indexes: &NewIndexes, operand: Operand| match operand { - Operand::Zero => Some(Counter::zero()), - Operand::Counter(id) => Some(Counter::counter_value_reference(id)), - Operand::Expression(id) => { - self.expressions - .get(id) - .expect("expression id is out of range") - .as_ref() - // If an expression was optimized out, assume it would have produced a count - // of zero. This ensures that expressions dependent on optimized-out - // expressions are still valid. - .map_or(Some(Counter::zero()), |_| new_indexes[id].map(Counter::expression)) - } - }; - - for (original_index, expression) in - self.expressions.iter_enumerated().filter_map(|(original_index, entry)| { - // Option::map() will return None to filter out missing expressions. This may happen - // if, for example, a MIR-instrumented expression is removed during an optimization. - entry.as_ref().map(|expression| (original_index, expression)) - }) - { - let optional_region = &expression.region; - let Expression { lhs, op, rhs, .. } = *expression; - - if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs) - .map(|lhs_counter| { - id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter)) - }) - { - if lhs_counter.is_zero() && op.is_subtract() { - // The left side of a subtraction was probably optimized out. As an example, - // a branch condition might be evaluated as a constant expression, and the - // branch could be removed, dropping unused counters in the process. - // - // Since counters are unsigned, we must assume the result of the expression - // can be no more and no less than zero. An expression known to evaluate to zero - // does not need to be added to the coverage map. - // - // Coverage test `loops_branches.rs` includes multiple variations of branches - // based on constant conditional (literal `true` or `false`), and demonstrates - // that the expected counts are still correct. - debug!( - "Expression subtracts from zero (assume unreachable): \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, lhs, op, rhs, optional_region, - ); - rhs_counter = Counter::zero(); - } - debug_assert!( - lhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((lhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "lhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - lhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - debug_assert!( - rhs_counter.is_zero() - // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` - || ((rhs_counter.zero_based_id() as usize) - <= usize::max(self.counters.len(), self.expressions.len())), - "rhs id={} > both counters.len()={} and expressions.len()={} - ({:?} {:?} {:?})", - rhs_counter.zero_based_id(), - self.counters.len(), - self.expressions.len(), - lhs_counter, - op, - rhs_counter, - ); - - // Both operands exist. `Expression` operands exist in `self.expressions` and have - // been assigned a `new_index`. - let mapped_expression_index = - MappedExpressionIndex::from(counter_expressions.len()); - let expression = CounterExpression::new( - lhs_counter, - match op { - Op::Add => ExprKind::Add, - Op::Subtract => ExprKind::Subtract, - }, - rhs_counter, - ); - debug!( - "Adding expression {:?} = {:?}, region: {:?}", - mapped_expression_index, expression, optional_region - ); - counter_expressions.push(expression); - new_indexes[original_index] = Some(mapped_expression_index); - if let Some(region) = optional_region { - expression_regions.push((Counter::expression(mapped_expression_index), region)); - } - } else { - bug!( - "expression has one or more missing operands \ - original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", - original_index, - lhs, - op, - rhs, - optional_region, - ); - } + fn counter_for_term(&self, term: CovTerm) -> Counter { + if is_zero_term(&self.counters_seen, &self.zero_expressions, term) { + Counter::ZERO + } else { + Counter::from_term(term) } - (counter_expressions, expression_regions.into_iter()) } +} + +/// Set of expression IDs that are known to always evaluate to zero. +/// Any mapping or expression operand that refers to these expressions can have +/// that reference replaced with a constant zero value. +#[derive(Default)] +struct ZeroExpressions(FxIndexSet<ExpressionId>); + +impl ZeroExpressions { + fn insert(&mut self, id: ExpressionId) { + self.0.insert(id); + } + + fn contains(&self, id: ExpressionId) -> bool { + self.0.contains(&id) + } +} - fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { - self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) +/// Returns `true` if the given term is known to have a value of zero, taking +/// into account knowledge of which counters are unused and which expressions +/// are always zero. +fn is_zero_term( + counters_seen: &BitSet<CounterId>, + zero_expressions: &ZeroExpressions, + term: CovTerm, +) -> bool { + match term { + CovTerm::Zero => true, + CovTerm::Counter(id) => !counters_seen.contains(id), + CovTerm::Expression(id) => zero_expressions.contains(id), } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 8ba7a11abe5..274e0aeaaba 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,18 +1,20 @@ use crate::common::CodegenCx; use crate::coverageinfo; use crate::coverageinfo::ffi::CounterMappingRegion; -use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::llvm; -use rustc_codegen_ssa::traits::ConstMethods; -use rustc_data_structures::fx::FxIndexSet; +use itertools::Itertools as _; +use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods}; +use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_index::IndexVec; use rustc_middle::bug; -use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir; use rustc_middle::mir::coverage::CodeRegion; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::def_id::DefIdSet; use rustc_span::Symbol; /// Generates and exports the Coverage Map. @@ -56,18 +58,40 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { return; } - let mut global_file_table = GlobalFileTable::new(tcx); + let function_coverage_entries = function_coverage_map + .into_iter() + .map(|(instance, function_coverage)| (instance, function_coverage.into_finished())) + .collect::<Vec<_>>(); + + let all_file_names = + function_coverage_entries.iter().flat_map(|(_, fn_cov)| fn_cov.all_file_names()); + let global_file_table = GlobalFileTable::new(all_file_names); + + // Encode all filenames referenced by coverage mappings in this CGU. + let filenames_buffer = global_file_table.make_filenames_buffer(tcx); + + let filenames_size = filenames_buffer.len(); + let filenames_val = cx.const_bytes(&filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); + + // Generate the coverage map header, which contains the filenames used by + // this CGU's coverage mappings, and store it in a well-known global. + let cov_data_val = generate_coverage_map(cx, version, filenames_size, filenames_val); + coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + + let mut unused_function_names = Vec::new(); + let covfun_section_name = coverageinfo::covfun_section_name(cx); // Encode coverage mappings and generate function records - let mut function_data = Vec::new(); - for (instance, function_coverage) in function_coverage_map { + for (instance, function_coverage) in function_coverage_entries { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); + let mangled_function_name = tcx.symbol_name(instance).name; let source_hash = function_coverage.source_hash(); let is_used = function_coverage.is_used(); let coverage_mapping_buffer = - encode_mappings_for_function(&mut global_file_table, &function_coverage); + encode_mappings_for_function(&global_file_table, &function_coverage); if coverage_mapping_buffer.is_empty() { if function_coverage.is_used() { @@ -81,21 +105,10 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { } } - function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); - } - - // Encode all filenames referenced by counters/expressions in this module - let filenames_buffer = global_file_table.into_filenames_buffer(); - - let filenames_size = filenames_buffer.len(); - let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); - - // Generate the LLVM IR representation of the coverage map and store it in a well-known global - let cov_data_val = generate_coverage_map(cx, version, filenames_size, filenames_val); + if !is_used { + unused_function_names.push(mangled_function_name); + } - let covfun_section_name = coverageinfo::covfun_section_name(cx); - for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { save_function_record( cx, &covfun_section_name, @@ -107,90 +120,143 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { ); } - // Save the coverage data value to LLVM IR - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + // For unused functions, we need to take their mangled names and store them + // in a specially-named global array. LLVM's `InstrProfiling` pass will + // detect this global and include those names in its `__llvm_prf_names` + // section. (See `llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp`.) + if !unused_function_names.is_empty() { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let name_globals = unused_function_names + .into_iter() + .map(|mangled_function_name| cx.const_str(mangled_function_name).0) + .collect::<Vec<_>>(); + let initializer = cx.const_array(cx.type_ptr(), &name_globals); + + let array = llvm::add_global(cx.llmod, cx.val_ty(initializer), "__llvm_coverage_names"); + llvm::set_global_constant(array, true); + llvm::set_linkage(array, llvm::Linkage::InternalLinkage); + llvm::set_initializer(array, initializer); + } } +/// Maps "global" (per-CGU) file ID numbers to their underlying filenames. struct GlobalFileTable { - global_file_table: FxIndexSet<Symbol>, + /// This "raw" table doesn't include the working dir, so a filename's + /// global ID is its index in this set **plus one**. + raw_file_table: FxIndexSet<Symbol>, } impl GlobalFileTable { - fn new(tcx: TyCtxt<'_>) -> Self { - let mut global_file_table = FxIndexSet::default(); + fn new(all_file_names: impl IntoIterator<Item = Symbol>) -> Self { + // Collect all of the filenames into a set. Filenames usually come in + // contiguous runs, so we can dedup adjacent ones to save work. + let mut raw_file_table = all_file_names.into_iter().dedup().collect::<FxIndexSet<Symbol>>(); + + // Sort the file table by its actual string values, not the arbitrary + // ordering of its symbols. + raw_file_table.sort_unstable_by(|a, b| a.as_str().cmp(b.as_str())); + + Self { raw_file_table } + } + + fn global_file_id_for_file_name(&self, file_name: Symbol) -> u32 { + let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| { + bug!("file name not found in prepared global file table: {file_name}"); + }); + // The raw file table doesn't include an entry for the working dir + // (which has ID 0), so add 1 to get the correct ID. + (raw_id + 1) as u32 + } + + fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> { // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) // requires setting the first filename to the compilation directory. // Since rustc generates coverage maps with relative paths, the // compilation directory can be combined with the relative paths // to get absolute paths, if needed. - let working_dir = Symbol::intern( - &tcx.sess.opts.working_dir.remapped_path_if_available().to_string_lossy(), - ); - global_file_table.insert(working_dir); - Self { global_file_table } - } - - fn global_file_id_for_file_name(&mut self, file_name: Symbol) -> u32 { - let (global_file_id, _) = self.global_file_table.insert_full(file_name); - global_file_id as u32 - } - - fn into_filenames_buffer(self) -> Vec<u8> { - // This method takes `self` so that the caller can't accidentally - // modify the original file table after encoding it into a buffer. + use rustc_session::RemapFileNameExt; + let working_dir: &str = &tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy(); llvm::build_byte_buffer(|buffer| { coverageinfo::write_filenames_section_to_buffer( - self.global_file_table.iter().map(Symbol::as_str), + // Insert the working dir at index 0, before the other filenames. + std::iter::once(working_dir).chain(self.raw_file_table.iter().map(Symbol::as_str)), buffer, ); }) } } +rustc_index::newtype_index! { + // Tell the newtype macro to not generate `Encode`/`Decode` impls. + #[custom_encodable] + struct LocalFileId {} +} + +/// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU) +/// file IDs. +#[derive(Default)] +struct VirtualFileMapping { + local_to_global: IndexVec<LocalFileId, u32>, + global_to_local: FxIndexMap<u32, LocalFileId>, +} + +impl VirtualFileMapping { + fn local_id_for_global(&mut self, global_file_id: u32) -> LocalFileId { + *self + .global_to_local + .entry(global_file_id) + .or_insert_with(|| self.local_to_global.push(global_file_id)) + } + + fn into_vec(self) -> Vec<u32> { + self.local_to_global.raw + } +} + /// Using the expressions and counter regions collected for a single function, /// generate the variable-sized payload of its corresponding `__llvm_covfun` /// entry. The payload is returned as a vector of bytes. /// /// Newly-encountered filenames will be added to the global file table. fn encode_mappings_for_function( - global_file_table: &mut GlobalFileTable, + global_file_table: &GlobalFileTable, function_coverage: &FunctionCoverage<'_>, ) -> Vec<u8> { - let (expressions, counter_regions) = function_coverage.get_expressions_and_counter_regions(); - - let mut counter_regions = counter_regions.collect::<Vec<_>>(); + let counter_regions = function_coverage.counter_regions(); if counter_regions.is_empty() { return Vec::new(); } - let mut virtual_file_mapping = IndexVec::<u32, u32>::new(); + let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); + + let mut virtual_file_mapping = VirtualFileMapping::default(); let mut mapping_regions = Vec::with_capacity(counter_regions.len()); - // Sort the list of (counter, region) mapping pairs by region, so that they - // can be grouped by filename. Prepare file IDs for each filename, and - // prepare the mapping data so that we can pass it through FFI to LLVM. - counter_regions.sort_by_key(|(_counter, region)| *region); - for counter_regions_for_file in - counter_regions.group_by(|(_, a), (_, b)| a.file_name == b.file_name) + // Group mappings into runs with the same filename, preserving the order + // yielded by `FunctionCoverage`. + // Prepare file IDs for each filename, and prepare the mapping data so that + // we can pass it through FFI to LLVM. + for (file_name, counter_regions_for_file) in + &counter_regions.group_by(|(_counter, region)| region.file_name) { - // Look up (or allocate) the global file ID for this filename. - let file_name = counter_regions_for_file[0].1.file_name; + // Look up the global file ID for this filename. let global_file_id = global_file_table.global_file_id_for_file_name(file_name); // Associate that global file ID with a local file ID for this function. - let local_file_id: u32 = virtual_file_mapping.push(global_file_id); - debug!(" file id: local {local_file_id} => global {global_file_id} = '{file_name:?}'"); + let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id); + debug!(" file id: {local_file_id:?} => global {global_file_id} = '{file_name:?}'"); // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. - for &(counter, region) in counter_regions_for_file { + for (counter, region) in counter_regions_for_file { let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = *region; debug!("Adding counter {counter:?} to map for {region:?}"); mapping_regions.push(CounterMappingRegion::code_region( counter, - local_file_id, + local_file_id.as_u32(), start_line, start_col, end_line, @@ -202,7 +268,7 @@ fn encode_mappings_for_function( // Encode the function's coverage mappings into a buffer. llvm::build_byte_buffer(|buffer| { coverageinfo::write_mapping_to_buffer( - virtual_file_mapping.raw, + virtual_file_mapping.into_vec(), expressions, mapping_regions, buffer, @@ -286,13 +352,12 @@ fn save_function_record( /// `-Clink-dead-code` will not generate code for unused generic functions.) /// /// We can find the unused functions (including generic functions) by the set difference of all MIR -/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query -/// `codegened_and_inlined_items`). +/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`codegenned_and_inlined_items`). /// -/// These unused functions are then codegen'd in one of the CGUs which is marked as the -/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating -/// code regions for the same function more than once which can lead to linker errors regarding -/// duplicate symbols. +/// These unused functions don't need to be codegenned, but we do need to add them to the function +/// coverage map (in a single designated CGU) so that we still emit coverage mappings for them. +/// We also end up adding their symbol names to a special global array that LLVM will include in +/// its embedded coverage data. fn add_unused_functions(cx: &CodegenCx<'_, '_>) { assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); @@ -312,7 +377,7 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { // generic functions from consideration as well. if !matches!( kind, - DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator + DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Coroutine ) { return None; } @@ -323,21 +388,80 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) { }) .collect(); - let codegenned_def_ids = tcx.codegened_and_inlined_items(()); + let codegenned_def_ids = codegenned_and_inlined_items(tcx); - for non_codegenned_def_id in - eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) - { - let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); - - // If a function is marked `#[coverage(off)]`, then skip generating a - // dead code stub for it. - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { - debug!("skipping unused fn marked #[coverage(off)]: {:?}", non_codegenned_def_id); + // For each `DefId` that should have coverage instrumentation but wasn't + // codegenned, add it to the function coverage map as an unused function. + for def_id in eligible_def_ids.into_iter().filter(|id| !codegenned_def_ids.contains(id)) { + // Skip any function that didn't have coverage data added to it by the + // coverage instrumentor. + let body = tcx.instance_mir(ty::InstanceDef::Item(def_id)); + let Some(function_coverage_info) = body.function_coverage_info.as_deref() else { continue; + }; + + debug!("generating unused fn: {def_id:?}"); + let instance = declare_unused_fn(tcx, def_id); + add_unused_function_coverage(cx, instance, function_coverage_info); + } +} + +/// All items participating in code generation together with (instrumented) +/// items inlined into them. +fn codegenned_and_inlined_items(tcx: TyCtxt<'_>) -> DefIdSet { + let (items, cgus) = tcx.collect_and_partition_mono_items(()); + let mut visited = DefIdSet::default(); + let mut result = items.clone(); + + for cgu in cgus { + for item in cgu.items().keys() { + if let mir::mono::MonoItem::Fn(ref instance) = item { + let did = instance.def_id(); + if !visited.insert(did) { + continue; + } + let body = tcx.instance_mir(instance.def); + for block in body.basic_blocks.iter() { + for statement in &block.statements { + let mir::StatementKind::Coverage(_) = statement.kind else { continue }; + let scope = statement.source_info.scope; + if let Some(inlined) = scope.inlined_instance(&body.source_scopes) { + result.insert(inlined.def_id()); + } + } + } + } } + } - debug!("generating unused fn: {:?}", non_codegenned_def_id); - cx.define_unused_fn(non_codegenned_def_id); + result +} + +fn declare_unused_fn<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> ty::Instance<'tcx> { + ty::Instance::new( + def_id, + ty::GenericArgs::for_item(tcx, def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ) +} + +fn add_unused_function_coverage<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + instance: ty::Instance<'tcx>, + function_coverage_info: &'tcx mir::coverage::FunctionCoverageInfo, +) { + // An unused function's mappings will automatically be rewritten to map to + // zero, because none of its counters/expressions are marked as seen. + let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); + + if let Some(coverage_context) = cx.coverage_context() { + coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); + } else { + bug!("Could not get the `coverage_context`"); } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index c70cb670e96..7d69756181a 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,10 +1,9 @@ use crate::llvm; -use crate::abi::Abi; use crate::builder::Builder; use crate::common::CodegenCx; use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; -use crate::coverageinfo::map_data::FunctionCoverage; +use crate::coverageinfo::map_data::FunctionCoverageCollector; use libc::c_uint; use rustc_codegen_ssa::traits::{ @@ -12,17 +11,12 @@ use rustc_codegen_ssa::traits::{ StaticMethods, }; use rustc_data_structures::fx::FxHashMap; -use rustc_hir as hir; -use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::{CounterId, CoverageKind}; +use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::Coverage; -use rustc_middle::ty; -use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; -use rustc_middle::ty::GenericArgs; +use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; -use rustc_middle::ty::Ty; use std::cell::RefCell; @@ -30,14 +24,13 @@ pub(crate) mod ffi; pub(crate) mod map_data; pub mod mapgen; -const UNUSED_FUNCTION_COUNTER_ID: CounterId = CounterId::START; - const VAR_ALIGN_BYTES: usize = 8; /// A context object for maintaining all state needed by the coverageinfo module. pub struct CrateCoverageContext<'ll, 'tcx> { /// Coverage data for each instrumented function identified by DefId. - pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>, + pub(crate) function_coverage_map: + RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, } @@ -49,7 +42,9 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } - pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> { + pub fn take_function_coverage_map( + &self, + ) -> FxHashMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> { self.function_coverage_map.replace(FxHashMap::default()) } } @@ -76,68 +71,56 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { bug!("Could not get the `coverage_context`"); } } - - /// Functions with MIR-based coverage are normally codegenned _only_ if - /// called. LLVM coverage tools typically expect every function to be - /// defined (even if unused), with at least one call to LLVM intrinsic - /// `instrprof.increment`. - /// - /// Codegen a small function that will never be called, with one counter - /// that will never be incremented. - /// - /// For used/called functions, the coverageinfo was already added to the - /// `function_coverage_map` (keyed by function `Instance`) during codegen. - /// But in this case, since the unused function was _not_ previously - /// codegenned, collect the coverage `CodeRegion`s from the MIR and add - /// them. The first `CodeRegion` is used to add a single counter, with the - /// same counter ID used in the injected `instrprof.increment` intrinsic - /// call. Since the function is never called, all other `CodeRegion`s can be - /// added as `unreachable_region`s. - fn define_unused_fn(&self, def_id: DefId) { - let instance = declare_unused_fn(self, def_id); - codegen_unused_fn_and_counter(self, instance); - add_unused_function_coverage(self, instance, def_id); - } } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { + // Our caller should have already taken care of inlining subtleties, + // so we can assume that counter/expression IDs in this coverage + // statement are meaningful for the given instance. + // + // (Either the statement was not inlined and directly belongs to this + // instance, or it was inlined *from* this instance.) + let bx = self; + let Some(function_coverage_info) = + bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() + else { + debug!("function has a coverage statement but no coverage info"); + return; + }; + let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map .entry(instance) - .or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance)); + .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); - let Coverage { kind, code_region } = coverage.clone(); - match kind { - CoverageKind::Counter { function_source_hash, id } => { - debug!( - "ensuring function source hash is set for instance={:?}; function_source_hash={}", - instance, function_source_hash, - ); - func_coverage.set_function_source_hash(function_source_hash); - - if let Some(code_region) = code_region { - // Note: Some counters do not have code regions, but may still be referenced - // from expressions. In that case, don't add the counter to the coverage map, - // but do inject the counter intrinsic. - debug!( - "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", - instance, id, code_region, - ); - func_coverage.add_counter(id, code_region); - } + let Coverage { kind } = coverage; + match *kind { + CoverageKind::CounterIncrement { id } => { + func_coverage.mark_counter_id_seen(id); // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`, // as that needs an exclusive borrow. drop(coverage_map); - let coverageinfo = bx.tcx().coverageinfo(instance.def); + // The number of counters passed to `llvm.instrprof.increment` might + // be smaller than the number originally inserted by the instrumentor, + // if some high-numbered counters were removed by MIR optimizations. + // If so, LLVM's profiler runtime will use fewer physical counters. + let num_counters = + bx.tcx().coverage_ids_info(instance.def).max_counter_id.as_u32() + 1; + assert!( + num_counters as usize <= function_coverage_info.num_counters, + "num_counters disagreement: query says {num_counters} but function info only has {}", + function_coverage_info.num_counters + ); let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(function_source_hash); - let num_counters = bx.const_u32(coverageinfo.num_counters); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let num_counters = bx.const_u32(num_counters); let index = bx.const_u32(id.as_u32()); debug!( "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", @@ -145,105 +128,13 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { ); bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::Expression { id, lhs, op, rhs } => { - debug!( - "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; region: {:?}", - instance, id, lhs, op, rhs, code_region, - ); - func_coverage.add_counter_expression(id, lhs, op, rhs, code_region); - } - CoverageKind::Unreachable => { - let code_region = - code_region.expect("unreachable regions always have code regions"); - debug!( - "adding unreachable code to coverage_map: instance={:?}, at {:?}", - instance, code_region, - ); - func_coverage.add_unreachable_region(code_region); + CoverageKind::ExpressionUsed { id } => { + func_coverage.mark_expression_id_seen(id); } } } } -fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { - let tcx = cx.tcx; - - let instance = Instance::new( - def_id, - GenericArgs::for_item(tcx, def_id, |param, _| { - if let ty::GenericParamDefKind::Lifetime = param.kind { - tcx.lifetimes.re_erased.into() - } else { - tcx.mk_param_from_def(param) - } - }), - ); - - let llfn = cx.declare_fn( - tcx.symbol_name(instance).name, - cx.fn_abi_of_fn_ptr( - ty::Binder::dummy(tcx.mk_fn_sig( - [Ty::new_unit(tcx)], - Ty::new_unit(tcx), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - )), - ty::List::empty(), - ), - None, - ); - - llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); - llvm::set_visibility(llfn, llvm::Visibility::Default); - - assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); - - instance -} - -fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { - let llfn = cx.get_fn(instance); - let llbb = Builder::append_block(cx, llfn, "unused_function"); - let mut bx = Builder::build(cx, llbb); - let fn_name = bx.get_pgo_func_name_var(instance); - let hash = bx.const_u64(0); - let num_counters = bx.const_u32(1); - let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, - index={:?}) for unused function: {:?}", - fn_name, hash, num_counters, index, instance - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); - bx.ret_void(); -} - -fn add_unused_function_coverage<'tcx>( - cx: &CodegenCx<'_, 'tcx>, - instance: Instance<'tcx>, - def_id: DefId, -) { - let tcx = cx.tcx; - - let mut function_coverage = FunctionCoverage::unused(tcx, instance); - for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { - if index == 0 { - // Insert at least one real counter so the LLVM CoverageMappingReader will find expected - // definitions. - function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); - } else { - function_coverage.add_unreachable_region(code_region.clone()); - } - } - - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } -} - /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 5ca2942ac4e..6a63eda4b99 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -75,7 +75,10 @@ fn make_mir_scope<'ll, 'tcx>( return; }; - if let Some(vars) = variables && !vars.contains(scope) && scope_data.inlined.is_none() { + if let Some(vars) = variables + && !vars.contains(scope) + && scope_data.inlined.is_none() + { // Do not create a DIScope if there are no variables defined in this // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. debug_context.scopes[scope] = parent_scope; @@ -90,7 +93,7 @@ fn make_mir_scope<'ll, 'tcx>( Some((callee, _)) => { // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. - let callee = cx.tcx.subst_and_normalize_erasing_regions( + let callee = cx.tcx.instantiate_and_normalize_erasing_regions( instance.args, ty::ParamEnv::reveal_all(), ty::EarlyBinder::bind(callee), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index ed938761694..865bf01c8c1 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -335,12 +335,20 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let (size, align) = match fn_ty.kind() { + ty::FnDef(..) => (0, 1), + ty::FnPtr(..) => ( + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + ), + _ => unreachable!(), + }; let di_node = unsafe { llvm::LLVMRustDIBuilderCreatePointerType( DIB(cx), fn_di_node, - cx.tcx.data_layout.pointer_size.bits(), - cx.tcx.data_layout.pointer_align.abi.bits() as u32, + size, + align, 0, // Ignore DWARF address space. name.as_ptr().cast(), name.len(), @@ -452,7 +460,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), - ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), + ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), AdtKind::Union => build_union_type_di_node(cx, unique_type_id), @@ -539,48 +547,77 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> ) -> &'ll DIFile { debug!(?source_file.name); + use rustc_session::RemapFileNameExt; let (directory, file_name) = match &source_file.name { FileName::Real(filename) => { let working_directory = &cx.sess().opts.working_dir; debug!(?working_directory); - let filename = cx - .sess() - .source_map() - .path_mapping() - .to_embeddable_absolute_path(filename.clone(), working_directory); - - // Construct the absolute path of the file - let abs_path = filename.remapped_path_if_available(); - debug!(?abs_path); - - if let Ok(rel_path) = - abs_path.strip_prefix(working_directory.remapped_path_if_available()) - { - // If the compiler's working directory (which also is the DW_AT_comp_dir of - // the compilation unit) is a prefix of the path we are about to emit, then - // only emit the part relative to the working directory. - // Because of path remapping we sometimes see strange things here: `abs_path` - // might actually look like a relative path - // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without - // taking the working directory into account, downstream tooling will - // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, - // which makes no sense. Usually in such cases the working directory will also - // be remapped to `<crate-name-and-version>` or some other prefix of the path - // we are remapping, so we end up with - // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. - // By moving the working directory portion into the `directory` part of the - // DIFile, we allow LLVM to emit just the relative path for DWARF, while - // still emitting the correct absolute path for CodeView. - ( - working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), - rel_path.to_string_lossy().into_owned(), - ) + if cx.sess().should_prefer_remapped_for_codegen() { + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `<crate-name-and-version>/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `<working-directory>/<crate-name-and-version>/src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `<crate-name-and-version>` or some other prefix of the path + // we are remapping, so we end up with + // `<crate-name-and-version>/<crate-name-and-version>/src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } } else { - ("".into(), abs_path.to_string_lossy().into_owned()) + let working_directory = working_directory.local_path_if_available(); + let filename = filename.local_path_if_available(); + + debug!(?working_directory, ?filename); + + let abs_path: Cow<'_, Path> = if filename.is_absolute() { + filename.into() + } else { + let mut p = PathBuf::new(); + p.push(working_directory); + p.push(filename); + p.into() + }; + + if let Ok(rel_path) = abs_path.strip_prefix(working_directory) { + ( + working_directory.to_string_lossy().into(), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } } } - other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + other => { + debug!(?other); + ("".into(), other.for_codegen(cx.sess()).to_string_lossy().into_owned()) + } }; let hash_kind = match source_file.src_hash.kind { @@ -814,8 +851,9 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. let producer = format!("clang LLVM ({rustc_producer})"); + use rustc_session::RemapFileNameExt; let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); - let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); + let work_dir = tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy(); let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { @@ -826,7 +864,13 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( Some(codegen_unit_name), ) // We get a path relative to the working directory from split_dwarf_path - .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) + .map(|f| { + if tcx.sess.should_prefer_remapped_for_split_debuginfo_paths() { + tcx.sess.source_map().path_mapping().map_prefix(f).0 + } else { + f.into() + } + }) } else { None } @@ -982,20 +1026,20 @@ fn build_struct_type_di_node<'ll, 'tcx>( // Tuples //=----------------------------------------------------------------------------- -/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. -/// For a generator, this will handle upvars shared by all states. +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or coroutine. +/// For a coroutine, this will handle upvars shared by all states. fn build_upvar_field_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - closure_or_generator_ty: Ty<'tcx>, - closure_or_generator_di_node: &'ll DIType, + closure_or_coroutine_ty: Ty<'tcx>, + closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { - let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { - ty::Generator(def_id, args, _) => (def_id, args.as_generator().prefix_tys()), + let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { + ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine().prefix_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), _ => { bug!( - "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", - closure_or_generator_ty + "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", + closure_or_coroutine_ty ) } }; @@ -1005,7 +1049,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( ); let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); - let layout = cx.layout_of(closure_or_generator_ty); + let layout = cx.layout_of(closure_or_coroutine_ty); up_var_tys .into_iter() @@ -1014,7 +1058,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( .map(|(index, (up_var_ty, capture_name))| { build_field_di_node( cx, - closure_or_generator_di_node, + closure_or_coroutine_di_node, capture_name.as_str(), cx.size_and_align_of(up_var_ty), layout.fields.offset(index), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 88040557a9b..ca7bfbeac25 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -12,7 +12,7 @@ use rustc_middle::{ ty::{ self, layout::{LayoutOf, TyAndLayout}, - AdtDef, GeneratorArgs, Ty, + AdtDef, CoroutineArgs, Ty, }, }; use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; @@ -268,18 +268,18 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ) } -/// A generator debuginfo node looks the same as a that of an enum type. +/// A coroutine debuginfo node looks the same as a that of an enum type. /// /// See [build_enum_type_di_node] for more information. -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let generator_type_and_layout = cx.layout_of(generator_type); - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + let coroutine_type = unique_type_id.expect_ty(); + let coroutine_type_and_layout = cx.layout_of(coroutine_type); + let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + debug_assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); type_map::build_type_with_children( cx, @@ -287,24 +287,24 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( cx, type_map::Stub::Union, unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), + &coroutine_type_name, + size_and_align_of(coroutine_type_and_layout), NO_SCOPE_METADATA, DIFlags::FlagZero, ), - |cx, generator_type_di_node| match generator_type_and_layout.variants { + |cx, coroutine_type_di_node| match coroutine_type_and_layout.variants { Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { - build_union_fields_for_direct_tag_generator( + build_union_fields_for_direct_tag_coroutine( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, ) } Variants::Single { .. } | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout + "Encountered coroutine with non-direct-tag layout: {:?}", + coroutine_type_and_layout ) } }, @@ -428,7 +428,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>( }) .collect(); - build_union_fields_for_direct_tag_enum_or_generator( + build_union_fields_for_direct_tag_enum_or_coroutine( cx, enum_type_and_layout, enum_type_di_node, @@ -469,8 +469,8 @@ fn build_variant_names_type_di_node<'ll, 'tcx>( fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_or_generator_type_and_layout: TyAndLayout<'tcx>, - enum_or_generator_type_di_node: &'ll DIType, + enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>, + enum_or_coroutine_type_di_node: &'ll DIType, variant_index: VariantIdx, untagged_variant_index: Option<VariantIdx>, variant_struct_type_di_node: &'ll DIType, @@ -486,13 +486,13 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( Stub::Struct, UniqueTypeId::for_enum_variant_struct_type_wrapper( cx.tcx, - enum_or_generator_type_and_layout.ty, + enum_or_coroutine_type_and_layout.ty, variant_index, ), &variant_struct_wrapper_type_name(variant_index), // NOTE: We use size and align of enum_type, not from variant_layout: - size_and_align_of(enum_or_generator_type_and_layout), - Some(enum_or_generator_type_di_node), + size_and_align_of(enum_or_coroutine_type_and_layout), + Some(enum_or_coroutine_type_di_node), DIFlags::FlagZero, ), |cx, wrapper_struct_type_di_node| { @@ -535,7 +535,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( cx, wrapper_struct_type_di_node, "value", - size_and_align_of(enum_or_generator_type_and_layout), + size_and_align_of(enum_or_coroutine_type_and_layout), Size::ZERO, DIFlags::FlagZero, variant_struct_type_di_node, @@ -662,40 +662,40 @@ fn split_128(value: u128) -> Split128 { Split128 { hi: (value >> 64) as u64, lo: value as u64 } } -fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( +fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, + coroutine_type_and_layout: TyAndLayout<'tcx>, + coroutine_type_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = - generator_type_and_layout.variants + coroutine_type_and_layout.variants else { bug!("This function only supports layouts with directly encoded tags.") }; - let (generator_def_id, generator_args) = match generator_type_and_layout.ty.kind() { - &ty::Generator(def_id, args, _) => (def_id, args.as_generator()), + let (coroutine_def_id, coroutine_args) = match coroutine_type_and_layout.ty.kind() { + &ty::Coroutine(def_id, args, _) => (def_id, args.as_coroutine()), _ => unreachable!(), }; - let generator_layout = cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); + let coroutine_layout = cx.tcx.optimized_mir(coroutine_def_id).coroutine_layout().unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); - let variant_range = generator_args.variant_range(generator_def_id, cx.tcx); + let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); + let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); - let tag_base_type = tag_base_type(cx, generator_type_and_layout); + let tag_base_type = tag_base_type(cx, coroutine_type_and_layout); let variant_names_type_di_node = build_variant_names_type_di_node( cx, - generator_type_di_node, + coroutine_type_di_node, variant_range .clone() - .map(|variant_index| (variant_index, GeneratorArgs::variant_name(variant_index))), + .map(|variant_index| (variant_index, CoroutineArgs::variant_name(variant_index))), ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { - let discriminants_iter = generator_args.discriminants(generator_def_id, cx.tcx); + let discriminants_iter = coroutine_args.discriminants(coroutine_def_id, cx.tcx); let mut discriminants: IndexVec<VariantIdx, DiscrResult> = IndexVec::with_capacity(variant_count); for (variant_index, discr) in discriminants_iter { @@ -709,16 +709,16 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( // Build the type node for each field. let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_range .map(|variant_index| { - let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( + let variant_struct_type_di_node = super::build_coroutine_variant_struct_type_di_node( cx, variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_layout, &common_upvar_names, ); - let span = generator_layout.variant_source_info[variant_index].span; + let span = coroutine_layout.variant_source_info[variant_index].span; let source_info = if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); Some((file_metadata(cx, &loc.file), loc.line as c_uint)) @@ -735,10 +735,10 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( }) .collect(); - build_union_fields_for_direct_tag_enum_or_generator( + build_union_fields_for_direct_tag_enum_or_coroutine( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, &variant_field_infos[..], variant_names_type_di_node, tag_base_type, @@ -747,9 +747,9 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( ) } -/// This is a helper function shared between enums and generators that makes sure fields have the +/// This is a helper function shared between enums and coroutines that makes sure fields have the /// expect names. -fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( +fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, enum_type_di_node: &'ll DIType, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index d3239d5c358..df1df6d197e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -6,11 +6,11 @@ use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::GeneratorLayout, + mir::CoroutineLayout, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - AdtDef, GeneratorArgs, Ty, VariantDef, + AdtDef, CoroutineArgs, Ty, VariantDef, }, }; use rustc_span::Symbol; @@ -66,14 +66,14 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( } } -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { if cpp_like_debuginfo(cx.tcx) { - cpp_like::build_generator_di_node(cx, unique_type_id) + cpp_like::build_coroutine_di_node(cx, unique_type_id) } else { - native::build_generator_di_node(cx, unique_type_id) + native::build_coroutine_di_node(cx, unique_type_id) } } @@ -101,13 +101,13 @@ fn build_c_style_enum_di_node<'ll, 'tcx>( } } -/// Extract the type with which we want to describe the tag of the given enum or generator. +/// Extract the type with which we want to describe the tag of the given enum or coroutine. fn tag_base_type<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, enum_type_and_layout: TyAndLayout<'tcx>, ) -> Ty<'tcx> { debug_assert!(match enum_type_and_layout.ty.kind() { - ty::Generator(..) => true, + ty::Coroutine(..) => true, ty::Adt(adt_def, _) => adt_def.is_enum(), _ => false, }); @@ -300,8 +300,8 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( .di_node } -/// Build the struct type for describing a single generator state. -/// See [build_generator_variant_struct_type_di_node]. +/// Build the struct type for describing a single coroutine state. +/// See [build_coroutine_variant_struct_type_di_node]. /// /// ```txt /// @@ -317,25 +317,25 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( /// ---> DW_TAG_structure_type (type of variant 3) /// /// ``` -pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( +pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, variant_index: VariantIdx, - generator_type_and_layout: TyAndLayout<'tcx>, - generator_type_di_node: &'ll DIType, - generator_layout: &GeneratorLayout<'tcx>, + coroutine_type_and_layout: TyAndLayout<'tcx>, + coroutine_type_di_node: &'ll DIType, + coroutine_layout: &CoroutineLayout<'tcx>, common_upvar_names: &IndexSlice<FieldIdx, Symbol>, ) -> &'ll DIType { - let variant_name = GeneratorArgs::variant_name(variant_index); + let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( cx.tcx, - generator_type_and_layout.ty, + coroutine_type_and_layout.ty, variant_index, ); - let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); + let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let generator_args = match generator_type_and_layout.ty.kind() { - ty::Generator(_, args, _) => args.as_generator(), + let coroutine_args = match coroutine_type_and_layout.ty.kind() { + ty::Coroutine(_, args, _) => args.as_coroutine(), _ => unreachable!(), }; @@ -346,17 +346,17 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( Stub::Struct, unique_type_id, &variant_name, - size_and_align_of(generator_type_and_layout), - Some(generator_type_di_node), + size_and_align_of(coroutine_type_and_layout), + Some(coroutine_type_di_node), DIFlags::FlagZero, ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) .map(|field_index| { - let generator_saved_local = generator_layout.variant_fields[variant_index] + let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; - let field_name_maybe = generator_layout.field_names[generator_saved_local]; + let field_name_maybe = coroutine_layout.field_names[coroutine_saved_local]; let field_name = field_name_maybe .as_ref() .map(|s| Cow::from(s.as_str())) @@ -377,7 +377,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .collect(); // Fields that are common to all states - let common_fields: SmallVec<_> = generator_args + let common_fields: SmallVec<_> = coroutine_args .prefix_tys() .iter() .zip(common_upvar_names) @@ -388,7 +388,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( variant_struct_type_di_node, upvar_name.as_str(), cx.size_and_align_of(upvar_ty), - generator_type_and_layout.fields.offset(index), + coroutine_type_and_layout.fields.offset(index), DIFlags::FlagZero, type_di_node(cx, upvar_ty), ) @@ -397,7 +397,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() }, - |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), + |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) .di_node } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index feac40d8c30..7eff52b857f 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -110,12 +110,12 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( ) } -/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for +/// Build the debuginfo node for a coroutine environment. It looks the same as the debuginfo for /// an enum. See [build_enum_type_di_node] for more information. /// /// ```txt /// -/// ---> DW_TAG_structure_type (top-level type for the generator) +/// ---> DW_TAG_structure_type (top-level type for the coroutine) /// DW_TAG_variant_part (variant part) /// DW_AT_discr (reference to discriminant DW_TAG_member) /// DW_TAG_member (discriminant member) @@ -127,21 +127,21 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( /// DW_TAG_structure_type (type of variant 3) /// /// ``` -pub(super) fn build_generator_di_node<'ll, 'tcx>( +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { - let generator_type = unique_type_id.expect_ty(); - let &ty::Generator(generator_def_id, _, _) = generator_type.kind() else { - bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) + let coroutine_type = unique_type_id.expect_ty(); + let &ty::Coroutine(coroutine_def_id, _, _) = coroutine_type.kind() else { + bug!("build_coroutine_di_node() called with non-coroutine type: `{:?}`", coroutine_type) }; - let containing_scope = get_namespace_for_item(cx, generator_def_id); - let generator_type_and_layout = cx.layout_of(generator_type); + let containing_scope = get_namespace_for_item(cx, coroutine_def_id); + let coroutine_type_and_layout = cx.layout_of(coroutine_type); - debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + debug_assert!(!wants_c_like_enum_debuginfo(coroutine_type_and_layout)); - let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); type_map::build_type_with_children( cx, @@ -149,37 +149,37 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( cx, Stub::Struct, unique_type_id, - &generator_type_name, - size_and_align_of(generator_type_and_layout), + &coroutine_type_name, + size_and_align_of(coroutine_type_and_layout), Some(containing_scope), DIFlags::FlagZero, ), - |cx, generator_type_di_node| { - let generator_layout = - cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); + |cx, coroutine_type_di_node| { + let coroutine_layout = + cx.tcx.optimized_mir(coroutine_def_id).coroutine_layout().unwrap(); let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = - generator_type_and_layout.variants + coroutine_type_and_layout.variants else { bug!( - "Encountered generator with non-direct-tag layout: {:?}", - generator_type_and_layout + "Encountered coroutine with non-direct-tag layout: {:?}", + coroutine_type_and_layout ) }; let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); + cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() .map(|variant_index| { // FIXME: This is problematic because just a number is not a valid identifier. - // GeneratorArgs::variant_name(variant_index), would be consistent + // CoroutineArgs::variant_name(variant_index), would be consistent // with enums? let variant_name = format!("{}", variant_index.as_usize()).into(); - let span = generator_layout.variant_source_info[variant_index].span; + let span = coroutine_layout.variant_source_info[variant_index].span; let source_info = if !span.is_dummy() { let loc = cx.lookup_debug_loc(span.lo()); Some((file_metadata(cx, &loc.file), loc.line)) @@ -191,12 +191,12 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( variant_index, variant_name, variant_struct_type_di_node: - super::build_generator_variant_struct_type_di_node( + super::build_coroutine_variant_struct_type_di_node( cx, variant_index, - generator_type_and_layout, - generator_type_di_node, - generator_layout, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_layout, &common_upvar_names, ), source_info, @@ -206,18 +206,18 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( smallvec![build_enum_variant_part_di_node( cx, - generator_type_and_layout, - generator_type_di_node, + coroutine_type_and_layout, + coroutine_type_di_node, &variant_struct_type_di_nodes[..], )] }, - // We don't seem to be emitting generic args on the generator type, it seems. Rather + // We don't seem to be emitting generic args on the coroutine type, it seems. Rather // they get attached to the struct type of each variant. NO_GENERICS, ) } -/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: +/// Builds the DW_TAG_variant_part of an enum or coroutine debuginfo node: /// /// ```txt /// DW_TAG_structure_type (top-level type for enum) @@ -306,11 +306,11 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( /// ``` fn build_discr_member_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, - enum_or_generator_type_and_layout: TyAndLayout<'tcx>, - enum_or_generator_type_di_node: &'ll DIType, + enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>, + enum_or_coroutine_type_di_node: &'ll DIType, ) -> Option<&'ll DIType> { - let tag_name = match enum_or_generator_type_and_layout.ty.kind() { - ty::Generator(..) => "__state", + let tag_name = match enum_or_coroutine_type_and_layout.ty.kind() { + ty::Coroutine(..) => "__state", _ => "", }; @@ -320,14 +320,14 @@ fn build_discr_member_di_node<'ll, 'tcx>( // In LLVM IR the wrong scope will be listed but when DWARF is // generated from it, the DW_TAG_member will be a child the // DW_TAG_variant_part. - let containing_scope = enum_or_generator_type_di_node; + let containing_scope = enum_or_coroutine_type_di_node; - match enum_or_generator_type_and_layout.layout.variants() { + match enum_or_coroutine_type_and_layout.layout.variants() { // A single-variant enum has no discriminant. &Variants::Single { .. } => None, &Variants::Multiple { tag_field, .. } => { - let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); + let tag_base_type = tag_base_type(cx, enum_or_coroutine_type_and_layout); let (size, align) = cx.size_and_align_of(tag_base_type); unsafe { @@ -340,7 +340,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( UNKNOWN_LINE_NUMBER, size.bits(), align.bits() as u32, - enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), + enum_or_coroutine_type_and_layout.fields.offset(tag_field).bits(), DIFlags::FlagArtificial, type_di_node(cx, tag_base_type), )) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index e30622cbdce..1aec65cf949 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -43,7 +43,7 @@ pub(super) enum UniqueTypeId<'tcx> { /// The ID of a regular type as it shows up at the language level. Ty(Ty<'tcx>, private::HiddenZst), /// The ID for the single DW_TAG_variant_part nested inside the top-level - /// DW_TAG_structure_type that describes enums and generators. + /// DW_TAG_structure_type that describes enums and coroutines. VariantPart(Ty<'tcx>, private::HiddenZst), /// The ID for the artificial struct type describing a single enum variant. VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index c862acdc7de..4832b147a54 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -50,7 +50,6 @@ mod utils; pub use self::create_scope_map::compute_mir_scopes; pub use self::metadata::build_global_var_di_node; -pub use self::metadata::extend_scope_to_file; #[allow(non_upper_case_globals)] const DW_TAG_auto_variable: c_uint = 0x100; @@ -342,7 +341,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // We look up the generics of the enclosing function and truncate the args // to their length in order to cut off extra stuff that might be in there for - // closures or generators. + // closures or coroutines. let generics = tcx.generics_of(enclosing_fn_def_id); let args = instance.args.truncate_to(tcx, generics); @@ -529,7 +528,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( instance.args, ty::ParamEnv::reveal_all(), cx.tcx.type_of(impl_def_id), @@ -537,7 +536,9 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // Only "class" methods are generally understood by LLVM, // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() && !def.is_box() { + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { // Again, only create type information if full debuginfo is enabled if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 264c273ba30..10ca5ad802a 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -26,6 +26,13 @@ pub(crate) struct UnknownCTargetFeature<'a> { pub rust_feature: PossibleFeature<'a>, } +#[derive(Diagnostic)] +#[diag(codegen_llvm_unstable_ctarget_feature)] +#[note] +pub(crate) struct UnstableCTargetFeature<'a> { + pub feature: &'a str, +} + #[derive(Subdiagnostic)] pub(crate) enum PossibleFeature<'a> { #[help(codegen_llvm_possible_feature)] @@ -139,6 +146,10 @@ pub(crate) struct LtoDisallowed; pub(crate) struct LtoDylib; #[derive(Diagnostic)] +#[diag(codegen_llvm_lto_proc_macro)] +pub(crate) struct LtoProcMacro; + +#[derive(Diagnostic)] #[diag(codegen_llvm_lto_bitcode_from_rlib)] pub(crate) struct LtoBitcodeFromRlib { pub llvm_err: String, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 9289c37d763..cc7e78b9c62 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -15,7 +15,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_hir as hir; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; @@ -376,7 +376,9 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } _ if name.as_str().starts_with("simd_") => { - match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { + match generic_simd_intrinsic( + self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, + ) { Ok(llval) => llval, Err(()) => return, } @@ -911,6 +913,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, + fn_args: GenericArgsRef<'tcx>, args: &[OperandRef<'tcx, &'ll Value>], ret_ty: Ty<'tcx>, llret_ty: &'ll Type, @@ -932,9 +935,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } macro_rules! require_simd { - ($ty: expr, $diag: expr) => { - require!($ty.is_simd(), $diag) - }; + ($ty: expr, $variant:ident) => {{ + require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty }); + $ty.simd_size_and_type(bx.tcx()) + }}; } let tcx = bx.tcx(); @@ -943,12 +947,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - require_simd!( - arg_tys[1], - InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } - ); - - let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (len, _) = require_simd!(arg_tys[1], SimdArgument); let expected_int_bits = (len.max(8) - 1).next_power_of_two(); let expected_bytes = len / 8 + ((len % 8 > 0) as u64); @@ -985,7 +984,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // every intrinsic below takes a SIMD vector as its first argument - require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] }); + let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput); let in_ty = arg_tys[0]; let comparison = match name { @@ -998,11 +997,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( _ => None, }; - let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); if let Some(cmp_op) = comparison { - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - - let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); require!( in_len == out_len, @@ -1030,6 +1026,55 @@ fn generic_simd_intrinsic<'ll, 'tcx>( )); } + if name == sym::simd_shuffle_generic { + let idx = fn_args[2] + .expect_const() + .eval(tcx, ty::ParamEnv::reveal_all(), Some(span)) + .unwrap() + .unwrap_branch(); + let n = idx.len() as u64; + + let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); + require!( + out_len == n, + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } + ); + require!( + in_elem == out_ty, + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } + ); + + let total_len = in_len * 2; + + let indices: Option<Vec<_>> = idx + .iter() + .enumerate() + .map(|(arg_idx, val)| { + let idx = val.unwrap_leaf().try_to_i32().unwrap(); + if idx >= i32::try_from(total_len).unwrap() { + bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { + span, + name, + arg_idx: arg_idx as u64, + total_len: total_len.into(), + }); + None + } else { + Some(bx.const_i32(idx)) + } + }) + .collect(); + let Some(indices) = indices else { + return Ok(bx.const_null(llret_ty)); + }; + + return Ok(bx.shuffle_vector( + args[0].immediate(), + args[1].immediate(), + bx.const_vector(&indices), + )); + } + if name == sym::simd_shuffle { // Make sure this is actually an array, since typeck only checks the length-suffixed // version of this intrinsic. @@ -1046,8 +1091,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }), }; - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); require!( out_len == n, InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } @@ -1126,11 +1170,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( if name == sym::simd_select { let m_elem_ty = in_elem; let m_len = in_len; - require_simd!( - arg_tys[1], - InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } - ); - let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (v_len, _) = require_simd!(arg_tys[1], SimdArgument); require!( m_len == v_len, InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } @@ -1348,20 +1388,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); - require_simd!( - arg_tys[1], - InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } - ); - require_simd!( - arg_tys[2], - InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } - ); - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); + + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = require_simd!(in_ty, SimdFirst); + let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond); + // The element type of the third argument must be a signed integer type of any width: + let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird); + require_simd!(ret_ty, SimdReturn); // Of the same length: - let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == out_len, InvalidMonomorphization::SecondArgumentLength { @@ -1391,11 +1427,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } ); - // The second argument must be a simd vector with an element type that's a pointer - // to the element type of the first argument - let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); - let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!( matches!( element_ty1.kind(), @@ -1412,20 +1443,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ); - // The element type of the third argument must be a signed integer type of any width: - let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); match element_ty2.kind() { ty::Int(_) => (), _ => { - require!( - false, - InvalidMonomorphization::ThirdArgElementType { - span, - name, - expected_element: element_ty2, - third_arg: arg_tys[2] - } - ); + return_error!(InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + }); } } @@ -1474,19 +1500,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // * M: any integer width is supported, will be truncated to i1 // All types must be simd vector types - require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); - require_simd!( - arg_tys[1], - InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } - ); - require_simd!( - arg_tys[2], - InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } - ); + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = require_simd!(in_ty, SimdFirst); + let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond); + let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird); // Of the same length: - let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); require!( in_len == element_len1, InvalidMonomorphization::SecondArgumentLength { @@ -1510,12 +1530,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } ); - // The second argument must be a simd vector with an element type that's a pointer - // to the element type of the first argument - let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); - let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); - let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); - require!( matches!( element_ty1.kind(), @@ -1537,15 +1551,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>( match element_ty2.kind() { ty::Int(_) => (), _ => { - require!( - false, - InvalidMonomorphization::ThirdArgElementType { - span, - name, - expected_element: element_ty2, - third_arg: arg_tys[2] - } - ); + return_error!(InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + }); } } @@ -1741,8 +1752,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( bitwise_red!(simd_reduce_any: vector_reduce_or, true); if name == sym::simd_cast_ptr { - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); require!( in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { @@ -1790,8 +1800,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_expose_addr { - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); require!( in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { @@ -1819,8 +1828,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_from_exposed_addr { - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); require!( in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { @@ -1848,8 +1856,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_cast || name == sym::simd_as { - require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); - let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn); require!( in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { @@ -1936,17 +1943,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } _ => { /* Unsupported. Fallthrough. */ } } - require!( - false, - InvalidMonomorphization::UnsupportedCast { - span, - name, - in_ty, - in_elem, - ret_ty, - out_elem - } - ); + return_error!(InvalidMonomorphization::UnsupportedCast { + span, + name, + in_ty, + in_elem, + ret_ty, + out_elem + }); } macro_rules! arith_binary { ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { @@ -1957,8 +1961,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( })* _ => {}, } - require!( - false, + return_error!( InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } ); })* @@ -1988,8 +1991,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( })* _ => {}, } - require!( - false, + return_error!( InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } ); })* diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ac199624e34..3242e78ab7e 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -4,13 +4,17 @@ //! //! This API is completely unstable and subject to change. +#![allow(internal_features)] +#![feature(rustdoc_internals)] +#![doc(rust_logo)] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(exact_size_is_empty)] #![feature(extern_types)] #![feature(hash_raw_entry)] #![feature(iter_intersperse)] #![feature(let_chains)] +#![feature(min_specialization)] #![feature(never_type)] -#![feature(slice_group_by)] #![feature(impl_trait_in_assoc_type)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] @@ -22,6 +26,7 @@ extern crate rustc_macros; #[macro_use] extern crate tracing; +use back::owned_target_machine::OwnedTargetMachine; use back::write::{create_informational_target_machine, create_target_machine}; use errors::ParseTargetMachineConfig; @@ -39,8 +44,8 @@ use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, Subd use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; -use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; +use rustc_middle::util::Providers; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; @@ -52,6 +57,7 @@ use std::io::Write; mod back { pub mod archive; pub mod lto; + pub mod owned_target_machine; mod profiling; pub mod write; } @@ -162,7 +168,7 @@ impl ExtraBackendMethods for LlvmCodegenBackend { impl WriteBackendMethods for LlvmCodegenBackend { type Module = ModuleLlvm; type ModuleBuffer = back::lto::ModuleBuffer; - type TargetMachine = &'static mut llvm::TargetMachine; + type TargetMachine = OwnedTargetMachine; type TargetMachineError = crate::errors::LlvmError<'static>; type ThinData = back::lto::ThinData; type ThinBuffer = back::lto::ThinBuffer; @@ -401,7 +407,9 @@ impl CodegenBackend for LlvmCodegenBackend { pub struct ModuleLlvm { llcx: &'static mut llvm::Context, llmod_raw: *const llvm::Module, - tm: &'static mut llvm::TargetMachine, + + // independent from llcx and llmod_raw, resources get disposed by drop impl + tm: OwnedTargetMachine, } unsafe impl Send for ModuleLlvm {} @@ -453,7 +461,6 @@ impl ModuleLlvm { impl Drop for ModuleLlvm { fn drop(&mut self) { unsafe { - llvm::LLVMRustDisposeTargetMachine(&mut *(self.tm as *mut _)); llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2ebfdae39e8..7fc02a95be0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -969,7 +969,6 @@ extern "C" { ConstantIndices: *const &'a Value, NumIndices: c_uint, ) -> &'a Value; - pub fn LLVMConstZExt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; @@ -2112,6 +2111,8 @@ extern "C" { ); pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + + // This function makes copies of pointed to data, so the data's lifetime may end after this function returns pub fn LLVMRustCreateTargetMachine( Triple: *const c_char, CPU: *const c_char, @@ -2131,13 +2132,14 @@ extern "C" { RelaxELFRelocations: bool, UseInitArray: bool, SplitDwarfFile: *const c_char, + OutputObjFile: *const c_char, DebugInfoCompression: *const c_char, ForceEmulatedTls: bool, ArgsCstrBuff: *const c_char, ArgsCstrBuffLen: usize, - ) -> Option<&'static mut TargetMachine>; + ) -> *mut TargetMachine; - pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); + pub fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index a76c9c9b735..c86bf81fc13 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -1,7 +1,7 @@ use crate::back::write::create_informational_target_machine; use crate::errors::{ PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, - UnknownCTargetFeaturePrefix, + UnknownCTargetFeaturePrefix, UnstableCTargetFeature, }; use crate::llvm; use libc::c_int; @@ -293,7 +293,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { supported_target_features(sess) .iter() .filter_map(|&(feature, gate)| { - if sess.is_nightly_build() || allow_unstable || gate.is_none() { + if sess.is_nightly_build() || allow_unstable || gate.is_stable() { Some(feature) } else { None @@ -303,7 +303,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> { // check that all features in a given smallvec are enabled for llvm_feature in to_llvm_features(sess, feature) { let cstr = SmallCStr::new(llvm_feature); - if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + if !unsafe { llvm::LLVMRustHasFeature(&target_machine, cstr.as_ptr()) } { return false; } } @@ -422,14 +422,14 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess } unsafe { llvm::LLVMRustPrintTargetCPUs( - tm, + &tm, cpu_cstring.as_ptr(), callback, &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, ); } } - PrintKind::TargetFeatures => print_target_features(out, sess, tm), + PrintKind::TargetFeatures => print_target_features(out, sess, &tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } } @@ -531,25 +531,35 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str }; let feature = backend_feature_name(s)?; - // Warn against use of LLVM specific feature names on the CLI. - if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) { - let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { - let llvm_features = to_llvm_features(sess, rust_feature); - if llvm_features.contains(&feature) && !llvm_features.contains(&rust_feature) { - Some(rust_feature) + // Warn against use of LLVM specific feature names and unstable features on the CLI. + if diagnostics { + let feature_state = supported_features.iter().find(|&&(v, _)| v == feature); + if feature_state.is_none() { + let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { + let llvm_features = to_llvm_features(sess, rust_feature); + if llvm_features.contains(&feature) + && !llvm_features.contains(&rust_feature) + { + Some(rust_feature) + } else { + None + } + }); + let unknown_feature = if let Some(rust_feature) = rust_feature { + UnknownCTargetFeature { + feature, + rust_feature: PossibleFeature::Some { rust_feature }, + } } else { - None - } - }); - let unknown_feature = if let Some(rust_feature) = rust_feature { - UnknownCTargetFeature { - feature, - rust_feature: PossibleFeature::Some { rust_feature }, - } - } else { - UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } - }; - sess.emit_warning(unknown_feature); + UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None } + }; + sess.emit_warning(unknown_feature); + } else if feature_state + .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable()) + { + // An unstable feature. Warn about using it. + sess.emit_warning(UnstableCTargetFeature { feature }); + } } if diagnostics { diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 38e8220569a..01e82339664 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -6,7 +6,6 @@ use crate::llvm; use crate::type_of::LayoutLlvmExt; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; -pub use rustc_middle::mir::mono::MonoItem; use rustc_middle::mir::mono::{Linkage, Visibility}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 8db6195d931..06b7703672f 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -112,7 +112,7 @@ impl<'ll> CodegenCx<'ll, '_> { } } - /// Return a LLVM type that has at most the required alignment, + /// Return an LLVM type that has at most the required alignment, /// and exactly the required size, as a best-effort padding array. pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { let unit = Integer::approximate_align(self, align); diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index dcc62d314ff..712b6ed5333 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -42,7 +42,7 @@ fn uncached_llvm_type<'a, 'tcx>( // FIXME(eddyb) producing readable type names for trait objects can result // in problematically distinct types due to HRTB and subtyping (see #47638). // ty::Dynamic(..) | - ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str // For performance reasons we use names only when emitting LLVM IR. if !cx.sess().fewer_names() => { @@ -54,10 +54,10 @@ fn uncached_llvm_type<'a, 'tcx>( write!(&mut name, "::{}", def.variant(index).name).unwrap(); } } - if let (&ty::Generator(_, _, _), &Variants::Single { index }) = + if let (&ty::Coroutine(_, _, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { - write!(&mut name, "::{}", ty::GeneratorArgs::variant_name(index)).unwrap(); + write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap(); } Some(name) } @@ -397,7 +397,12 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // extracts all the individual values. let ety = element.llvm_type(cx); - return Some(cx.type_vector(ety, *count)); + if *count == 1 { + // Emitting `<1 x T>` would be silly; just use the scalar. + return Some(ety); + } else { + return Some(cx.type_vector(ety, *count)); + } } // FIXME: The above only handled integer arrays; surely more things |