diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
34 files changed, 1178 insertions, 336 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 28be6d033f8..d221bad28ef 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -351,7 +351,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { continue; } PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { - let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); + let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty); let ptr_layout = cx.layout_of(ptr_ty); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index ad0636894b7..a57508815d6 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -77,7 +77,7 @@ pub(crate) unsafe fn codegen( llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); @@ -129,7 +129,7 @@ pub(crate) unsafe fn codegen( attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); - let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()); + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index 6d00464e0a0..e4fce30e232 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,7 +1,6 @@ //! Set and unset common attributes on LLVM values. use rustc_codegen_ssa::traits::*; -use rustc_data_structures::small_str::SmallStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::{self, TyCtxt}; @@ -336,6 +335,10 @@ pub fn from_fn_attrs<'ll, 'tcx>( to_add.extend(probestack_attr(cx)); to_add.extend(stackprotector_attr(cx)); + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { + to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); } @@ -360,50 +363,44 @@ pub fn from_fn_attrs<'ll, 'tcx>( if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); - let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; - if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { - flags |= AllocKindFlags::Uninitialized; - } else { - flags |= AllocKindFlags::Zeroed; - } - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); // apply to return place instead of function (unlike all other attributes applied in this function) let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr( - cx.llcx, - AllocKindFlags::Realloc | AllocKindFlags::Aligned, - )); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - // apply to argument place instead of function - let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); - to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); - } + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { - if llvm_util::get_version() >= (15, 0, 0) { - to_add.push(create_alloc_family_attr(cx.llcx)); - to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); - // applies to argument place instead of function place - let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); - attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); - } + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); } if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); @@ -481,8 +478,8 @@ pub fn from_fn_attrs<'ll, 'tcx>( let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); let function_features = function_features.iter().map(|s| s.as_str()); - let target_features = - global_features.chain(function_features).intersperse(",").collect::<SmallStr<1024>>(); + let target_features: String = + global_features.chain(function_features).intersperse(",").collect(); if !target_features.is_empty() { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index a6416e9540c..10bf954d242 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -198,25 +198,24 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { "arm" => ("arm", "--32"), _ => panic!("unsupported arch {}", sess.target.arch), }; - let result = std::process::Command::new(&dlltool) - .args([ - "-d", - def_file_path.to_str().unwrap(), - "-D", - lib_name, - "-l", - output_path.to_str().unwrap(), - "-m", - dlltool_target_arch, - "-f", - dlltool_target_bitness, - "--no-leading-underscore", - "--temp-prefix", - temp_prefix.to_str().unwrap(), - ]) - .output(); - - match result { + let mut dlltool_cmd = std::process::Command::new(&dlltool); + dlltool_cmd.args([ + "-d", + def_file_path.to_str().unwrap(), + "-D", + lib_name, + "-l", + output_path.to_str().unwrap(), + "-m", + dlltool_target_arch, + "-f", + dlltool_target_bitness, + "--no-leading-underscore", + "--temp-prefix", + temp_prefix.to_str().unwrap(), + ]); + + match dlltool_cmd.output() { Err(e) => { sess.emit_fatal(ErrorCallingDllTool { dlltool_path: dlltool.to_string_lossy(), @@ -226,6 +225,12 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { // dlltool returns '0' on failure, so check for error output instead. Ok(output) if !output.stderr.is_empty() => { sess.emit_fatal(DlltoolFailImportLibrary { + dlltool_path: dlltool.to_string_lossy(), + dlltool_args: dlltool_cmd + .get_args() + .map(|arg| arg.to_string_lossy()) + .collect::<Vec<_>>() + .join(" "), stdout: String::from_utf8_lossy(&output.stdout), stderr: String::from_utf8_lossy(&output.stderr), }) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 8b05af7bed9..d7dd98d7938 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -1,4 +1,4 @@ -use crate::back::write::{self, save_temp_bitcode, DiagnosticHandlers}; +use crate::back::write::{self, save_temp_bitcode, CodegenDiagnosticsStage, DiagnosticHandlers}; use crate::errors::{ DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, }; @@ -302,7 +302,13 @@ fn fat_lto( // The linking steps below may produce errors and diagnostics within LLVM // which we'd like to handle and print, so set up our diagnostic handlers // (which get unregistered when they go out of scope below). - let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handler = DiagnosticHandlers::new( + cgcx, + diag_handler, + llcx, + &module, + CodegenDiagnosticsStage::LTO, + ); // For all other modules we codegened we'll need to link them into our own // bitcode. All modules were codegened in their own LLVM context, however, @@ -595,7 +601,7 @@ pub(crate) fn run_pass_manager( llvm::LLVMRustAddModuleFlag( module.module_llvm.llmod(), llvm::LLVMModFlagBehavior::Error, - c"LTOPostLink".as_ptr().cast(), + "LTOPostLink\0".as_ptr().cast(), 1, ); } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 53b4296802e..0f5e975445f 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -268,6 +268,16 @@ pub(crate) fn save_temp_bitcode( } } +/// In what context is a dignostic handler being attached to a codegen unit? +pub enum CodegenDiagnosticsStage { + /// Prelink optimization stage. + Opt, + /// LTO/ThinLTO postlink optimization stage. + LTO, + /// Code generation. + Codegen, +} + pub struct DiagnosticHandlers<'a> { data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a Handler), llcx: &'a llvm::Context, @@ -279,6 +289,8 @@ impl<'a> DiagnosticHandlers<'a> { cgcx: &'a CodegenContext<LlvmCodegenBackend>, handler: &'a Handler, llcx: &'a llvm::Context, + module: &ModuleCodegen<ModuleLlvm>, + stage: CodegenDiagnosticsStage, ) -> Self { let remark_passes_all: bool; let remark_passes: Vec<CString>; @@ -295,6 +307,20 @@ impl<'a> DiagnosticHandlers<'a> { }; let remark_passes: Vec<*const c_char> = remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); + let remark_file = cgcx + .remark_dir + .as_ref() + // Use the .opt.yaml file suffix, which is supported by LLVM's opt-viewer. + .map(|dir| { + let stage_suffix = match stage { + CodegenDiagnosticsStage::Codegen => "codegen", + CodegenDiagnosticsStage::Opt => "opt", + CodegenDiagnosticsStage::LTO => "lto", + }; + dir.join(format!("{}.{stage_suffix}.opt.yaml", module.name)) + }) + .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok())); + let data = Box::into_raw(Box::new((cgcx, handler))); unsafe { let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); @@ -305,6 +331,9 @@ impl<'a> DiagnosticHandlers<'a> { remark_passes_all, remark_passes.as_ptr(), remark_passes.len(), + // The `as_ref()` is important here, otherwise the `CString` will be dropped + // too soon! + remark_file.as_ref().map(|dir| dir.as_ptr()).unwrap_or(std::ptr::null()), ); DiagnosticHandlers { data, llcx, old_handler } } @@ -523,7 +552,8 @@ pub(crate) unsafe fn optimize( let llmod = module.module_llvm.llmod(); let llcx = &*module.module_llvm.llcx; - let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handlers = + DiagnosticHandlers::new(cgcx, diag_handler, llcx, module, CodegenDiagnosticsStage::Opt); let module_name = module.name.clone(); let module_name = Some(&module_name[..]); @@ -582,7 +612,13 @@ pub(crate) unsafe fn codegen( let tm = &*module.module_llvm.tm; let module_name = module.name.clone(); let module_name = Some(&module_name[..]); - let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + let _handlers = DiagnosticHandlers::new( + cgcx, + diag_handler, + llcx, + &module, + CodegenDiagnosticsStage::Codegen, + ); if cgcx.msvc_imps_needed { create_msvc_imps(cgcx, llcx, llmod); @@ -775,7 +811,6 @@ pub(crate) unsafe fn codegen( } record_llvm_cgu_instructions_stats(&cgcx.prof, llmod); - drop(handlers); } // `.dwo` files are only emitted if: @@ -875,14 +910,19 @@ unsafe fn embed_bitcode( // passed though then these sections will show up in the final output. // Additionally the flag that we need to set here is `SHF_EXCLUDE`. // + // * XCOFF - AIX linker ignores content in .ipa and .info if no auxiliary + // symbol associated with these sections. + // // Unfortunately, LLVM provides no way to set custom section flags. For ELF // and COFF we emit the sections using module level inline assembly for that // reason (see issue #90326 for historical background). + let is_aix = cgcx.opts.target_triple.triple().contains("-aix"); let is_apple = cgcx.opts.target_triple.triple().contains("-ios") || cgcx.opts.target_triple.triple().contains("-darwin") || cgcx.opts.target_triple.triple().contains("-tvos") || cgcx.opts.target_triple.triple().contains("-watchos"); if is_apple + || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") || cgcx.opts.target_triple.triple().starts_with("asmjs") { @@ -891,11 +931,17 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - c"rustc.embedded.module".as_ptr().cast(), + "rustc.embedded.module\0".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { c"__LLVM,__bitcode" } else { c".llvmbc" }; + let section = if is_apple { + "__LLVM,__bitcode\0" + } else if is_aix { + ".ipa\0" + } else { + ".llvmbc\0" + }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); @@ -904,10 +950,16 @@ unsafe fn embed_bitcode( let llglobal = llvm::LLVMAddGlobal( llmod, common::val_ty(llconst), - c"rustc.embedded.cmdline".as_ptr().cast(), + "rustc.embedded.cmdline\0".as_ptr().cast(), ); llvm::LLVMSetInitializer(llglobal, llconst); - let section = if is_apple { c"__LLVM,__cmdline" } else { c".llvmcmd" }; + let section = if is_apple { + "__LLVM,__cmdline\0" + } else if is_aix { + ".info\0" + } else { + ".llvmcmd\0" + }; llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } else { diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 2f7eb08ad3d..5b5f81c0329 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -19,6 +19,8 @@ use crate::context::CodegenCx; use crate::llvm; use crate::value::Value; +use cstr::cstr; + use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; @@ -84,8 +86,8 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen { let cx = CodegenCx::new(tcx, cgu, &llvm_module); let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); - for &(mono_item, (linkage, visibility)) in &mono_items { - mono_item.predefine::<Builder<'_, '_, '_>>(&cx, linkage, visibility); + for &(mono_item, data) in &mono_items { + mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility); } // ... and now that we have everything pre-defined, fill out those definitions. @@ -108,11 +110,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen // Create the llvm.used and llvm.compiler.used variables. if !cx.used_statics.borrow().is_empty() { - cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow()); + cx.create_used_variable_impl(cstr!("llvm.used"), &*cx.used_statics.borrow()); } if !cx.compiler_used_statics.borrow().is_empty() { cx.create_used_variable_impl( - c"llvm.compiler.used", + cstr!("llvm.compiler.used"), &*cx.compiler_used_statics.borrow(), ); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index b4aa001547c..d55992bf092 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -6,6 +6,7 @@ use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True} use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; +use cstr::cstr; use libc::{c_char, c_uint}; use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; @@ -23,7 +24,9 @@ use rustc_span::Span; use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; +use smallvec::SmallVec; use std::borrow::Cow; +use std::ffi::CStr; use std::iter; use std::ops::Deref; use std::ptr; @@ -43,10 +46,13 @@ impl Drop for Builder<'_, '_, '_> { } } +// FIXME(eddyb) use a checked constructor when they become `const fn`. +const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; + /// Empty string, to be used where LLVM expects an instruction name, indicating /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. -const UNNAMED: *const c_char = c"".as_ptr(); +const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value; @@ -225,7 +231,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("invoke", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); - let mut bundles = vec![funclet_bundle]; + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } // Emit CFI pointer type membership test self.cfi_type_test(fn_attrs, fn_abi, llfn); @@ -233,9 +242,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } - bundles.retain(|bundle| bundle.is_some()); let invoke = unsafe { llvm::LLVMRustBuildInvoke( self.llbuilder, @@ -572,8 +582,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ) { let zero = self.const_usize(0); let count = self.const_usize(count); - let start = dest.project_index(self, zero).llval; - let end = dest.project_index(self, count).llval; let header_bb = self.append_sibling_block("repeat_loop_header"); let body_bb = self.append_sibling_block("repeat_loop_body"); @@ -582,24 +590,18 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.br(header_bb); let mut header_bx = Self::build(self.cx, header_bb); - let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); + let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]); - let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); + let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count); header_bx.cond_br(keep_going, body_bb, next_bb); let mut body_bx = Self::build(self.cx, body_bb); - let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); - cg_elem - .val - .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); - - let next = body_bx.inbounds_gep( - self.backend_type(cg_elem.layout), - current, - &[self.const_usize(1)], - ); + let dest_elem = dest.project_index(&mut body_bx, i); + cg_elem.val.store(&mut body_bx, dest_elem); + + let next = body_bx.unchecked_uadd(i, self.const_usize(1)); body_bx.br(header_bb); - header_bx.add_incoming_to_phi(current, next, body_bb); + header_bx.add_incoming_to_phi(i, next, body_bb); *self = Self::build(self.cx, next_bb); } @@ -1005,13 +1007,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("cleanuppad"); let ret = unsafe { llvm::LLVMBuildCleanupPad( self.llbuilder, parent, args.as_ptr(), args.len() as c_uint, - c"cleanuppad".as_ptr(), + name.as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) @@ -1025,13 +1028,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("catchpad"); let ret = unsafe { llvm::LLVMBuildCatchPad( self.llbuilder, parent, args.as_ptr(), args.len() as c_uint, - c"catchpad".as_ptr(), + name.as_ptr(), ) }; Funclet::new(ret.expect("LLVM does not have support for catchpad")) @@ -1043,13 +1047,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unwind: Option<&'ll BasicBlock>, handlers: &[&'ll BasicBlock], ) -> &'ll Value { + let name = cstr!("catchswitch"); let ret = unsafe { llvm::LLVMBuildCatchSwitch( self.llbuilder, parent, unwind, handlers.len() as c_uint, - c"catchswitch".as_ptr(), + name.as_ptr(), ) }; let ret = ret.expect("LLVM does not have support for catchswitch"); @@ -1189,7 +1194,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let args = self.check_call("call", llty, llfn, args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); - let mut bundles = vec![funclet_bundle]; + let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); + if let Some(funclet_bundle) = funclet_bundle { + bundles.push(funclet_bundle); + } // Emit CFI pointer type membership test self.cfi_type_test(fn_attrs, fn_abi, llfn); @@ -1197,9 +1205,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Emit KCFI operand bundle let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn); let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); - bundles.push(kcfi_bundle); + if let Some(kcfi_bundle) = kcfi_bundle { + bundles.push(kcfi_bundle); + } - bundles.retain(|bundle| bundle.is_some()); let call = unsafe { llvm::LLVMRustBuildCall( self.llbuilder, diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 4b9ca2e7d19..5d3e245f45f 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -27,8 +27,8 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> debug!("get_fn(instance={:?})", instance); - assert!(!instance.substs.has_infer()); - assert!(!instance.substs.has_escaping_bound_vars()); + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); if let Some(&llfn) = cx.instances.borrow().get(&instance) { return llfn; @@ -129,7 +129,7 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - let is_generic = instance.substs.non_erasable_generics().next().is_some(); + let is_generic = instance.args.non_erasable_generics().next().is_some(); if is_generic { // This is a monomorphization. Its expected visibility depends diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index a3910fef954..4d879fbdcb7 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -168,6 +168,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.const_uint(self.type_i64(), i) } + fn const_u128(&self, i: u128) -> &'ll Value { + self.const_uint_big(self.type_i128(), i) + } + fn const_usize(&self, i: u64) -> &'ll Value { let bit_size = self.data_layout().pointer_size.bits(); if bit_size < 64 { @@ -286,7 +290,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { } }; let llval = unsafe { - llvm::LLVMRustConstInBoundsGEP2( + llvm::LLVMConstInBoundsGEP2( self.type_i8(), self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)), &self.const_usize(offset.bytes()), @@ -316,7 +320,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { unsafe { - llvm::LLVMRustConstInBoundsGEP2( + llvm::LLVMConstInBoundsGEP2( self.type_i8(), self.const_bitcast(base_addr, self.type_i8p()), &self.const_usize(offset.bytes()), diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 2087754c66b..df52f50f86f 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -8,6 +8,7 @@ use crate::llvm::{self, True}; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; use crate::value::Value; +use cstr::cstr; use rustc_codegen_ssa::traits::*; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; @@ -481,9 +482,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { .all(|&byte| byte == 0); let sect_name = if all_bytes_are_zero { - c"__DATA,__thread_bss" + cstr!("__DATA,__thread_bss") } else { - c"__DATA,__thread_data" + cstr!("__DATA,__thread_data") }; llvm::LLVMSetSection(g, sect_name.as_ptr()); } @@ -512,7 +513,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> { let val = llvm::LLVMMetadataAsValue(self.llcx, meta); llvm::LLVMAddNamedMetadataOperand( self.llmod, - c"wasm.custom_sections".as_ptr().cast(), + "wasm.custom_sections\0".as_ptr().cast(), val, ); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index e8a7afcc632..5bf641055c5 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,8 @@ use crate::llvm_util; use crate::type_::Type; use crate::value::Value; -use rustc_codegen_ssa::base::wants_msvc_seh; +use cstr::cstr; +use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n; use rustc_data_structures::fx::FxHashMap; @@ -32,6 +33,7 @@ use rustc_target::abi::{ use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; use smallvec::SmallVec; +use libc::c_uint; use std::cell::{Cell, RefCell}; use std::ffi::CStr; use std::str; @@ -223,42 +225,36 @@ pub unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"RtLibUseGOT".as_ptr().cast(), - 1, - ); + let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); } // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { + let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - c"CFI Canonical Jump Tables".as_ptr().cast(), + canonical_jump_tables, 1, ); } // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { + let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast(); llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - c"EnableSplitLTOUnit".as_ptr().cast(), + enable_split_lto_unit, 1, ); } // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - llvm::LLVMRustAddModuleFlag( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"kcfi".as_ptr().cast(), - 1, - ); + let kcfi = "kcfi\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); } // Control Flow Guard is currently only supported by the MSVC linker on Windows. @@ -270,7 +266,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, + "cfguard\0".as_ptr() as *const _, 1, ) } @@ -279,7 +275,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, + "cfguard\0".as_ptr() as *const _, 2, ) } @@ -297,26 +293,26 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, behavior, - c"branch-target-enforcement".as_ptr().cast(), + "branch-target-enforcement\0".as_ptr().cast(), bti.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - c"sign-return-address".as_ptr().cast(), + "sign-return-address\0".as_ptr().cast(), pac_ret.is_some().into(), ); let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); llvm::LLVMRustAddModuleFlag( llmod, behavior, - c"sign-return-address-all".as_ptr().cast(), + "sign-return-address-all\0".as_ptr().cast(), pac_opts.leaf.into(), ); llvm::LLVMRustAddModuleFlag( llmod, behavior, - c"sign-return-address-with-bkey".as_ptr().cast(), + "sign-return-address-with-bkey\0".as_ptr().cast(), u32::from(pac_opts.key == PAuthKey::B), ); } else { @@ -332,7 +328,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - c"cf-protection-branch".as_ptr().cast(), + "cf-protection-branch\0".as_ptr().cast(), 1, ) } @@ -340,7 +336,7 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Override, - c"cf-protection-return".as_ptr().cast(), + "cf-protection-return\0".as_ptr().cast(), 1, ) } @@ -349,11 +345,28 @@ pub unsafe fn create_module<'ll>( llvm::LLVMRustAddModuleFlag( llmod, llvm::LLVMModFlagBehavior::Error, - c"Virtual Function Elim".as_ptr().cast(), + "Virtual Function Elim\0".as_ptr().cast(), 1, ); } + // Insert `llvm.ident` metadata. + // + // On the wasm targets it will get hooked up to the "producer" sections + // `processed-by` information. + let rustc_producer = + format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); + let name_metadata = llvm::LLVMMDStringInContext( + llcx, + rustc_producer.as_ptr().cast(), + rustc_producer.as_bytes().len() as c_uint, + ); + llvm::LLVMAddNamedMetadataOperand( + llmod, + cstr!("llvm.ident").as_ptr(), + llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1), + ); + llmod } @@ -481,13 +494,14 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { + let section = cstr!("llvm.metadata"); let array = self.const_array(self.type_ptr_to(self.type_i8()), values); unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); - llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); + llvm::LLVMSetSection(g, section.as_ptr()); } } } @@ -532,19 +546,28 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { if let Some(llpersonality) = self.eh_personality.get() { return llpersonality; } + + let name = if wants_msvc_seh(self.sess()) { + Some("__CxxFrameHandler3") + } else if wants_wasm_eh(self.sess()) { + // LLVM specifically tests for the name of the personality function + // There is no need for this function to exist anywhere, it will + // not be called. However, its name has to be "__gxx_wasm_personality_v0" + // for native wasm exceptions. + Some("__gxx_wasm_personality_v0") + } else { + None + }; + let tcx = self.tcx; let llfn = match tcx.lang_items().eh_personality() { - Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( + Some(def_id) if name.is_none() => self.get_fn_addr( ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty()) .unwrap() .unwrap(), ), _ => { - let name = if wants_msvc_seh(self.sess()) { - "__CxxFrameHandler3" - } else { - "rust_eh_personality" - }; + let name = name.unwrap_or("rust_eh_personality"); if let Some(llfn) = self.get_declared_value(name) { llfn } else { @@ -662,6 +685,10 @@ impl<'ll> CodegenCx<'ll, '_> { let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); let t_metadata = self.type_metadata(); + let t_token = self.type_token(); + + ifn!("llvm.wasm.get.exception", fn(t_token) -> i8p); + ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs new file mode 100644 index 00000000000..1791ce4b315 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -0,0 +1,89 @@ +use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; + +/// Must match the layout of `LLVMRustCounterKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum CounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +} + +/// A reference to an instance of an abstract "counter" that will yield a value in a coverage +/// report. Note that `id` has different interpretations, depending on the `kind`: +/// * For `CounterKind::Zero`, `id` is assumed to be `0` +/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected +/// instrumentation counter (the `index` argument to the LLVM intrinsic +/// `instrprof.increment()`) +/// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of +/// counter expressions. +/// +/// Corresponds to struct `llvm::coverage::Counter`. +/// +/// Must match the layout of `LLVMRustCounter`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct Counter { + // Important: The layout (order and types of fields) must match its C++ counterpart. + pub kind: CounterKind, + id: u32, +} + +impl Counter { + /// Constructs a new `Counter` of kind `Zero`. For this `CounterKind`, the + /// `id` is not used. + pub fn zero() -> Self { + Self { kind: CounterKind::Zero, id: 0 } + } + + /// Constructs a new `Counter` of kind `CounterValueReference`, and converts + /// the given 1-based counter_id to the required 0-based equivalent for + /// the `Counter` encoding. + pub fn counter_value_reference(counter_id: CounterValueReference) -> Self { + Self { kind: CounterKind::CounterValueReference, id: counter_id.zero_based_index() } + } + + /// Constructs a new `Counter` of kind `Expression`. + pub fn expression(mapped_expression_index: MappedExpressionIndex) -> Self { + Self { kind: CounterKind::Expression, id: mapped_expression_index.into() } + } + + /// Returns true if the `Counter` kind is `Zero`. + pub fn is_zero(&self) -> bool { + matches!(self.kind, CounterKind::Zero) + } + + /// An explicitly-named function to get the ID value, making it more obvious + /// that the stored value is now 0-based. + pub fn zero_based_id(&self) -> u32 { + debug_assert!(!self.is_zero(), "`id` is undefined for CounterKind::Zero"); + self.id + } +} + +/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`. +/// +/// Must match the layout of `LLVMRustCounterExprKind`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub enum ExprKind { + Subtract = 0, + Add = 1, +} + +/// Corresponds to struct `llvm::coverage::CounterExpression`. +/// +/// Must match the layout of `LLVMRustCounterExpression`. +#[derive(Copy, Clone, Debug)] +#[repr(C)] +pub struct CounterExpression { + pub kind: ExprKind, + pub lhs: Counter, + pub rhs: Counter, +} + +impl CounterExpression { + pub fn new(lhs: Counter, kind: ExprKind, rhs: Counter) -> Self { + Self { kind, lhs, rhs } + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs new file mode 100644 index 00000000000..06844afd6b8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs @@ -0,0 +1,348 @@ +pub use super::ffi::*; + +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, + InjectedExpressionIndex, MappedExpressionIndex, Op, +}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::TyCtxt; + +#[derive(Clone, Debug, PartialEq)] +pub struct Expression { + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option<CodeRegion>, +} + +/// Collects all of the coverage regions associated with (a) injected counters, (b) counter +/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero), +/// for a given Function. Counters and counter expressions have non-overlapping `id`s because they +/// can both be operands in an expression. This struct also stores the `function_source_hash`, +/// computed during instrumentation, and forwarded with counters. +/// +/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap +/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter +/// or expression), but the line or lines in the gap region are not executable (such as lines with +/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count +/// for a gap area is only used as the line execution count if there are no other regions on a +/// line." +#[derive(Debug)] +pub struct FunctionCoverage<'tcx> { + instance: Instance<'tcx>, + source_hash: u64, + is_used: bool, + counters: IndexVec<CounterValueReference, Option<CodeRegion>>, + expressions: IndexVec<InjectedExpressionIndex, Option<Expression>>, + unreachable_regions: Vec<CodeRegion>, +} + +impl<'tcx> FunctionCoverage<'tcx> { + /// Creates a new set of coverage data for a used (called) function. + pub fn new(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, true) + } + + /// Creates a new set of coverage data for an unused (never called) function. + pub fn unused(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { + Self::create(tcx, instance, false) + } + + fn create(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, is_used: bool) -> Self { + let coverageinfo = tcx.coverageinfo(instance.def); + debug!( + "FunctionCoverage::create(instance={:?}) has coverageinfo={:?}. is_used={}", + instance, coverageinfo, is_used + ); + Self { + instance, + source_hash: 0, // will be set with the first `add_counter()` + is_used, + counters: IndexVec::from_elem_n(None, coverageinfo.num_counters as usize), + expressions: IndexVec::from_elem_n(None, coverageinfo.num_expressions as usize), + unreachable_regions: Vec::new(), + } + } + + /// Returns true for a used (called) function, and false for an unused function. + pub fn is_used(&self) -> bool { + self.is_used + } + + /// Sets the function source hash value. If called multiple times for the same function, all + /// calls should have the same hash value. + pub fn set_function_source_hash(&mut self, source_hash: u64) { + if self.source_hash == 0 { + self.source_hash = source_hash; + } else { + debug_assert_eq!(source_hash, self.source_hash); + } + } + + /// Adds a code region to be counted by an injected counter intrinsic. + pub fn add_counter(&mut self, id: CounterValueReference, region: CodeRegion) { + if let Some(previous_region) = self.counters[id].replace(region.clone()) { + assert_eq!(previous_region, region, "add_counter: code region for id changed"); + } + } + + /// Both counters and "counter expressions" (or simply, "expressions") can be operands in other + /// expressions. Expression IDs start from `u32::MAX` and go down, so the range of expression + /// IDs will not overlap with the range of counter IDs. Counters and expressions can be added in + /// any order, and expressions can still be assigned contiguous (though descending) IDs, without + /// knowing what the last counter ID will be. + /// + /// When storing the expression data in the `expressions` vector in the `FunctionCoverage` + /// struct, its vector index is computed, from the given expression ID, by subtracting from + /// `u32::MAX`. + /// + /// Since the expression operands (`lhs` and `rhs`) can reference either counters or + /// expressions, an operand that references an expression also uses its original ID, descending + /// from `u32::MAX`. Theses operands are translated only during code generation, after all + /// counters and expressions have been added. + pub fn add_counter_expression( + &mut self, + expression_id: InjectedExpressionId, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option<CodeRegion>, + ) { + debug!( + "add_counter_expression({:?}, lhs={:?}, op={:?}, rhs={:?} at {:?}", + expression_id, lhs, op, rhs, region + ); + let expression_index = self.expression_index(u32::from(expression_id)); + debug_assert!( + expression_index.as_usize() < self.expressions.len(), + "expression_index {} is out of range for expressions.len() = {} + for {:?}", + expression_index.as_usize(), + self.expressions.len(), + self, + ); + if let Some(previous_expression) = self.expressions[expression_index].replace(Expression { + lhs, + op, + rhs, + region: region.clone(), + }) { + assert_eq!( + previous_expression, + Expression { lhs, op, rhs, region }, + "add_counter_expression: expression for id changed" + ); + } + } + + /// Add a region that will be marked as "unreachable", with a constant "zero counter". + pub fn add_unreachable_region(&mut self, region: CodeRegion) { + self.unreachable_regions.push(region) + } + + /// Return the source hash, generated from the HIR node structure, and used to indicate whether + /// or not the source code structure changed between different compilations. + pub fn source_hash(&self) -> u64 { + self.source_hash + } + + /// Generate an array of CounterExpressions, and an iterator over all `Counter`s and their + /// associated `Regions` (from which the LLVM-specific `CoverageMapGenerator` will create + /// `CounterMappingRegion`s. + pub fn get_expressions_and_counter_regions( + &self, + ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { + assert!( + self.source_hash != 0 || !self.is_used, + "No counters provided the source_hash for used function: {:?}", + self.instance + ); + + let counter_regions = self.counter_regions(); + let (counter_expressions, expression_regions) = self.expressions_with_regions(); + let unreachable_regions = self.unreachable_regions(); + + let counter_regions = + counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions)); + (counter_expressions, counter_regions) + } + + fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { + self.counters.iter_enumerated().filter_map(|(index, entry)| { + // Option::map() will return None to filter out missing counters. This may happen + // if, for example, a MIR-instrumented counter is removed during an optimization. + entry.as_ref().map(|region| (Counter::counter_value_reference(index), region)) + }) + } + + fn expressions_with_regions( + &self, + ) -> (Vec<CounterExpression>, impl Iterator<Item = (Counter, &CodeRegion)>) { + let mut counter_expressions = Vec::with_capacity(self.expressions.len()); + let mut expression_regions = Vec::with_capacity(self.expressions.len()); + let mut new_indexes = IndexVec::from_elem_n(None, self.expressions.len()); + + // This closure converts any `Expression` operand (`lhs` or `rhs` of the `Op::Add` or + // `Op::Subtract` operation) into its native `llvm::coverage::Counter::CounterKind` type + // and value. Operand ID value `0` maps to `CounterKind::Zero`; values in the known range + // of injected LLVM counters map to `CounterKind::CounterValueReference` (and the value + // matches the injected counter index); and any other value is converted into a + // `CounterKind::Expression` with the expression's `new_index`. + // + // Expressions will be returned from this function in a sequential vector (array) of + // `CounterExpression`, so the expression IDs must be mapped from their original, + // potentially sparse set of indexes, originally in reverse order from `u32::MAX`. + // + // An `Expression` as an operand will have already been encountered as an `Expression` with + // operands, so its new_index will already have been generated (as a 1-up index value). + // (If an `Expression` as an operand does not have a corresponding new_index, it was + // probably optimized out, after the expression was injected into the MIR, so it will + // get a `CounterKind::Zero` instead.) + // + // In other words, an `Expression`s at any given index can include other expressions as + // operands, but expression operands can only come from the subset of expressions having + // `expression_index`s lower than the referencing `Expression`. Therefore, it is + // reasonable to look up the new index of an expression operand while the `new_indexes` + // vector is only complete up to the current `ExpressionIndex`. + let id_to_counter = |new_indexes: &IndexSlice< + InjectedExpressionIndex, + Option<MappedExpressionIndex>, + >, + id: ExpressionOperandId| { + if id == ExpressionOperandId::ZERO { + Some(Counter::zero()) + } else if id.index() < self.counters.len() { + debug_assert!( + id.index() > 0, + "ExpressionOperandId indexes for counters are 1-based, but this id={}", + id.index() + ); + // Note: Some codegen-injected Counters may be only referenced by `Expression`s, + // and may not have their own `CodeRegion`s, + let index = CounterValueReference::from(id.index()); + // Note, the conversion to LLVM `Counter` adjusts the index to be zero-based. + Some(Counter::counter_value_reference(index)) + } else { + let index = self.expression_index(u32::from(id)); + self.expressions + .get(index) + .expect("expression id is out of range") + .as_ref() + // If an expression was optimized out, assume it would have produced a count + // of zero. This ensures that expressions dependent on optimized-out + // expressions are still valid. + .map_or(Some(Counter::zero()), |_| new_indexes[index].map(Counter::expression)) + } + }; + + for (original_index, expression) in + self.expressions.iter_enumerated().filter_map(|(original_index, entry)| { + // Option::map() will return None to filter out missing expressions. This may happen + // if, for example, a MIR-instrumented expression is removed during an optimization. + entry.as_ref().map(|expression| (original_index, expression)) + }) + { + let optional_region = &expression.region; + let Expression { lhs, op, rhs, .. } = *expression; + + if let Some(Some((lhs_counter, mut rhs_counter))) = id_to_counter(&new_indexes, lhs) + .map(|lhs_counter| { + id_to_counter(&new_indexes, rhs).map(|rhs_counter| (lhs_counter, rhs_counter)) + }) + { + if lhs_counter.is_zero() && op.is_subtract() { + // The left side of a subtraction was probably optimized out. As an example, + // a branch condition might be evaluated as a constant expression, and the + // branch could be removed, dropping unused counters in the process. + // + // Since counters are unsigned, we must assume the result of the expression + // can be no more and no less than zero. An expression known to evaluate to zero + // does not need to be added to the coverage map. + // + // Coverage test `loops_branches.rs` includes multiple variations of branches + // based on constant conditional (literal `true` or `false`), and demonstrates + // that the expected counts are still correct. + debug!( + "Expression subtracts from zero (assume unreachable): \ + original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", + original_index, lhs, op, rhs, optional_region, + ); + rhs_counter = Counter::zero(); + } + debug_assert!( + lhs_counter.is_zero() + // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` + || ((lhs_counter.zero_based_id() as usize) + <= usize::max(self.counters.len(), self.expressions.len())), + "lhs id={} > both counters.len()={} and expressions.len()={} + ({:?} {:?} {:?})", + lhs_counter.zero_based_id(), + self.counters.len(), + self.expressions.len(), + lhs_counter, + op, + rhs_counter, + ); + + debug_assert!( + rhs_counter.is_zero() + // Note: with `as usize` the ID _could_ overflow/wrap if `usize = u16` + || ((rhs_counter.zero_based_id() as usize) + <= usize::max(self.counters.len(), self.expressions.len())), + "rhs id={} > both counters.len()={} and expressions.len()={} + ({:?} {:?} {:?})", + rhs_counter.zero_based_id(), + self.counters.len(), + self.expressions.len(), + lhs_counter, + op, + rhs_counter, + ); + + // Both operands exist. `Expression` operands exist in `self.expressions` and have + // been assigned a `new_index`. + let mapped_expression_index = + MappedExpressionIndex::from(counter_expressions.len()); + let expression = CounterExpression::new( + lhs_counter, + match op { + Op::Add => ExprKind::Add, + Op::Subtract => ExprKind::Subtract, + }, + rhs_counter, + ); + debug!( + "Adding expression {:?} = {:?}, region: {:?}", + mapped_expression_index, expression, optional_region + ); + counter_expressions.push(expression); + new_indexes[original_index] = Some(mapped_expression_index); + if let Some(region) = optional_region { + expression_regions.push((Counter::expression(mapped_expression_index), region)); + } + } else { + bug!( + "expression has one or more missing operands \ + original_index={:?}, lhs={:?}, op={:?}, rhs={:?}, region={:?}", + original_index, + lhs, + op, + rhs, + optional_region, + ); + } + } + (counter_expressions, expression_regions.into_iter()) + } + + fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> { + self.unreachable_regions.iter().map(|region| (Counter::zero(), region)) + } + + fn expression_index(&self, id_descending_from_max: u32) -> InjectedExpressionIndex { + debug_assert!(id_descending_from_max >= self.counters.len() as u32); + InjectedExpressionIndex::from(u32::MAX - id_descending_from_max) + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 21a1ac34844..9149b06886b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,10 +1,10 @@ use crate::common::CodegenCx; use crate::coverageinfo; +use crate::coverageinfo::map_data::{Counter, CounterExpression}; use crate::llvm; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; -use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; +use rustc_codegen_ssa::traits::ConstMethods; use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; @@ -63,7 +63,7 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { let mut function_data = Vec::new(); for (instance, function_coverage) in function_coverage_map { debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); - let mangled_function_name = tcx.symbol_name(instance).to_string(); + let mangled_function_name = tcx.symbol_name(instance).name; let source_hash = function_coverage.source_hash(); let is_used = function_coverage.is_used(); let (expressions, counter_regions) = @@ -95,14 +95,16 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) { let filenames_size = filenames_buffer.len(); let filenames_val = cx.const_bytes(&filenames_buffer); - let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(&filenames_buffer); // Generate the LLVM IR representation of the coverage map and store it in a well-known global let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); + let covfun_section_name = coverageinfo::covfun_section_name(cx); for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { save_function_record( cx, + &covfun_section_name, mangled_function_name, source_hash, filenames_ref, @@ -228,7 +230,8 @@ impl CoverageMapGenerator { /// specific, well-known section and name. fn save_function_record( cx: &CodegenCx<'_, '_>, - mangled_function_name: String, + covfun_section_name: &str, + mangled_function_name: &str, source_hash: u64, filenames_ref: u64, coverage_mapping_buffer: Vec<u8>, @@ -238,7 +241,7 @@ fn save_function_record( let coverage_mapping_size = coverage_mapping_buffer.len(); let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); - let func_name_hash = coverageinfo::hash_str(&mangled_function_name); + let func_name_hash = coverageinfo::hash_bytes(mangled_function_name.as_bytes()); let func_name_hash_val = cx.const_u64(func_name_hash); let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); let source_hash_val = cx.const_u64(source_hash); @@ -254,7 +257,13 @@ fn save_function_record( /*packed=*/ true, ); - coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); + coverageinfo::save_func_record_to_mod( + cx, + covfun_section_name, + func_name_hash, + func_record_val, + is_used, + ); } /// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index cd261293e9b..60cbb3d3d9d 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -3,13 +3,13 @@ use crate::llvm; use crate::abi::Abi; use crate::builder::Builder; use crate::common::CodegenCx; +use crate::coverageinfo::map_data::{CounterExpression, FunctionCoverage}; use libc::c_uint; use llvm::coverageinfo::CounterMappingRegion; -use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; use rustc_codegen_ssa::traits::{ - BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, - MiscMethods, StaticMethods, + BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods, + StaticMethods, }; use rustc_data_structures::fx::FxHashMap; use rustc_hir as hir; @@ -17,16 +17,20 @@ use rustc_hir::def_id::DefId; use rustc_llvm::RustString; use rustc_middle::bug; use rustc_middle::mir::coverage::{ - CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, + CodeRegion, CounterValueReference, CoverageKind, ExpressionOperandId, InjectedExpressionId, Op, }; +use rustc_middle::mir::Coverage; use rustc_middle::ty; -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::subst::InternalSubsts; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::GenericArgs; use rustc_middle::ty::Instance; +use rustc_middle::ty::Ty; use std::cell::RefCell; use std::ffi::CString; +mod ffi; +pub(crate) mod map_data; pub mod mapgen; const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; @@ -53,11 +57,17 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } -impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn coverageinfo_finalize(&self) { +// These methods used to be part of trait `CoverageInfoMethods`, which no longer +// exists after most coverage code was moved out of SSA. +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + pub(crate) fn coverageinfo_finalize(&self) { mapgen::finalize(self) } + /// For LLVM codegen, returns a function-specific `Value` for a global + /// string, to hold the function name passed to LLVM intrinsic + /// `instrprof.increment()`. The `Value` is only created once per instance. + /// Multiple invocations with the same instance return the same `Value`. fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { if let Some(coverage_context) = self.coverage_context() { debug!("getting pgo_func_name_var for instance={:?}", instance); @@ -94,6 +104,54 @@ impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { + let bx = self; + + let Coverage { kind, code_region } = coverage.clone(); + match kind { + CoverageKind::Counter { function_source_hash, id } => { + if bx.set_function_source_hash(instance, function_source_hash) { + // If `set_function_source_hash()` returned true, the coverage map is enabled, + // so continue adding the counter. + if let Some(code_region) = code_region { + // Note: Some counters do not have code regions, but may still be referenced + // from expressions. In that case, don't add the counter to the coverage map, + // but do inject the counter intrinsic. + bx.add_coverage_counter(instance, id, code_region); + } + + let coverageinfo = bx.tcx().coverageinfo(instance.def); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_source_hash); + let num_counters = bx.const_u32(coverageinfo.num_counters); + let index = bx.const_u32(id.zero_based_index()); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + } + } + CoverageKind::Expression { id, lhs, op, rhs } => { + bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); + } + CoverageKind::Unreachable => { + bx.add_coverage_unreachable( + instance, + code_region.expect("unreachable regions always have code regions"), + ); + } + } + } +} + +// These methods used to be part of trait `CoverageInfoBuilderMethods`, but +// after moving most coverage code out of SSA they are now just ordinary methods. +impl<'tcx> Builder<'_, '_, 'tcx> { + /// Returns true if the function source hash was added to the coverage map (even if it had + /// already been added, for this instance). Returns false *only* if `-C instrument-coverage` is + /// not enabled (a coverage map is not being generated). fn set_function_source_hash( &mut self, instance: Instance<'tcx>, @@ -115,6 +173,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } + /// Returns true if the counter was added to the coverage map; false if `-C instrument-coverage` + /// is not enabled (a coverage map is not being generated). fn add_coverage_counter( &mut self, instance: Instance<'tcx>, @@ -137,6 +197,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } + /// Returns true if the expression was added to the coverage map; false if + /// `-C instrument-coverage` is not enabled (a coverage map is not being generated). fn add_coverage_counter_expression( &mut self, instance: Instance<'tcx>, @@ -163,6 +225,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } } + /// Returns true if the region was added to the coverage map; false if `-C instrument-coverage` + /// is not enabled (a coverage map is not being generated). fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { if let Some(coverage_context) = self.coverage_context() { debug!( @@ -186,7 +250,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< let instance = Instance::new( def_id, - InternalSubsts::for_item(tcx, def_id, |param, _| { + GenericArgs::for_item(tcx, def_id, |param, _| { if let ty::GenericParamDefKind::Lifetime = param.kind { tcx.lifetimes.re_erased.into() } else { @@ -199,8 +263,8 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< tcx.symbol_name(instance).name, cx.fn_abi_of_fn_ptr( ty::Binder::dummy(tcx.mk_fn_sig( - [tcx.mk_unit()], - tcx.mk_unit(), + [Ty::new_unit(tcx)], + Ty::new_unit(tcx), false, hir::Unsafety::Unsafe, Abi::Rust, @@ -309,12 +373,7 @@ pub(crate) fn write_mapping_to_buffer( } } -pub(crate) fn hash_str(strval: &str) -> u64 { - let strval = CString::new(strval).expect("null error converting hashable str to C string"); - unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } -} - -pub(crate) fn hash_bytes(bytes: Vec<u8>) -> u64 { +pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } } @@ -349,6 +408,7 @@ pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, + covfun_section_name: &str, func_name_hash: u64, func_record_val: &'ll llvm::Value, is_used: bool, @@ -364,20 +424,33 @@ pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( let func_record_var_name = format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); debug!("function record var name: {:?}", func_record_var_name); - - let func_record_section_name = llvm::build_string(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - }) - .expect("Rust Coverage function record section name failed UTF-8 conversion"); - debug!("function record section name: {:?}", func_record_section_name); + debug!("function record section name: {:?}", covfun_section_name); let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); llvm::set_initializer(llglobal, func_record_val); llvm::set_global_constant(llglobal, true); llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, &func_record_section_name); + llvm::set_section(llglobal, covfun_section_name); llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); cx.add_used_global(llglobal); } + +/// Returns the section name string to pass through to the linker when embedding +/// per-function coverage information in the object file, according to the target +/// platform's object file format. +/// +/// LLVM's coverage tools read coverage mapping details from this section when +/// producing coverage reports. +/// +/// Typical values are: +/// - `__llvm_covfun` on Linux +/// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) +/// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) +pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String { + llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage function record section name failed UTF-8 conversion") +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs index 64961baf272..d174a3593b9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -65,10 +65,10 @@ fn make_mir_scope<'ll, 'tcx>( debug_context.scopes[parent] } else { // The root is the function itself. - let loc = cx.lookup_debug_loc(mir.span.lo()); + let file = cx.sess().source_map().lookup_source_file(mir.span.lo()); debug_context.scopes[scope] = DebugScope { - file_start_pos: loc.file.start_pos, - file_end_pos: loc.file.end_pos, + file_start_pos: file.start_pos, + file_end_pos: file.end_pos, ..debug_context.scopes[scope] }; instantiated.insert(scope); @@ -91,7 +91,7 @@ fn make_mir_scope<'ll, 'tcx>( // FIXME(eddyb) this would be `self.monomorphize(&callee)` // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. let callee = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, + instance.args, ty::ParamEnv::reveal_all(), ty::EarlyBinder::bind(callee), ); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 8be54b7eb71..37f30917609 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -38,6 +38,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; section_var.unwrap_or_else(|| { + let section_name = b".debug_gdb_scripts\0"; let mut section_contents = Vec::new(); // Add the pretty printers for the standard library first. @@ -70,7 +71,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, ' let section_var = cx .define_global(section_var_name, llvm_type) .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); - llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast()); + llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 166454d3ae7..905e0e541a8 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -20,6 +20,7 @@ use crate::llvm::debuginfo::{ }; use crate::value::Value; +use cstr::cstr; use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; use rustc_codegen_ssa::traits::*; @@ -167,7 +168,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // a (fat) pointer. Make sure it is not called for e.g. `Box<T, NonZSTAllocator>`. debug_assert_eq!( cx.size_and_align_of(ptr_type), - cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) + cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type)) ); let pointee_type_di_node = type_di_node(cx, pointee_type); @@ -222,8 +223,11 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // at all and instead emit regular struct debuginfo for it. We just // need to make sure that we don't break existing debuginfo consumers // by doing that (at least not without a warning period). - let layout_type = - if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; + let layout_type = if ptr_type.is_box() { + Ty::new_mut_ptr(cx.tcx, pointee_type) + } else { + ptr_type + }; let layout = cx.layout_of(layout_type); let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); @@ -429,7 +433,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D return existing_di_node; } - debug!("type_di_node: {:?}", t); + debug!("type_di_node: {:?} kind: {:?}", t, t.kind()); let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { @@ -445,7 +449,7 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D } // Box<T, A> may have a non-ZST allocator A. In that case, we // cannot treat Box<T, A> as just an owned alias of `*mut T`. - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_zst() => { build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) } ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), @@ -735,7 +739,10 @@ fn build_foreign_type_di_node<'ll, 'tcx>( debug!("build_foreign_type_di_node: {:?}", t); let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { - bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); + bug!( + "build_foreign_type_di_node() called with unexpected type: {:?}", + unique_type_id.expect_ty() + ); }; build_type_with_children( @@ -811,6 +818,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); + let flags = "\0"; let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames @@ -847,7 +855,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( producer.as_ptr().cast(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, - c"".as_ptr().cast(), + flags.as_ptr().cast(), 0, // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final @@ -876,22 +884,8 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>( ); let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata); - llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val); - } - - // Insert `llvm.ident` metadata on the wasm targets since that will - // get hooked up to the "producer" sections `processed-by` information. - if tcx.sess.target.is_like_wasm { - let name_metadata = llvm::LLVMMDStringInContext( - debug_context.llcontext, - rustc_producer.as_ptr().cast(), - rustc_producer.as_bytes().len() as c_uint, - ); - llvm::LLVMAddNamedMetadataOperand( - debug_context.llmod, - c"llvm.ident".as_ptr(), - llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), - ); + let llvm_gcov_ident = cstr!("llvm.gcov"); + llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val); } return unit_metadata; @@ -998,12 +992,12 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_generator_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { - ty::Generator(def_id, substs, _) => { - let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); + ty::Generator(def_id, args, _) => { + let upvar_tys: SmallVec<_> = args.as_generator().prefix_tys().collect(); (def_id, upvar_tys) } - ty::Closure(def_id, substs) => { - let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); + ty::Closure(def_id, args) => { + let upvar_tys: SmallVec<_> = args.as_closure().upvar_tys().collect(); (def_id, upvar_tys) } _ => { @@ -1031,7 +1025,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( build_field_di_node( cx, closure_or_generator_di_node, - capture_name, + capture_name.as_str(), cx.size_and_align_of(up_var_ty), layout.fields.offset(index), DIFlags::FlagZero, @@ -1093,7 +1087,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let closure_env_type = unique_type_id.expect_ty(); - let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { + let &ty::Closure(def_id, _args) = closure_env_type.kind() else { bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) }; let containing_scope = get_namespace_for_item(cx, def_id); @@ -1171,11 +1165,11 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, ) -> SmallVec<&'ll DIType> { - if let ty::Adt(def, substs) = *ty.kind() { - if substs.types().next().is_some() { + if let ty::Adt(def, args) = *ty.kind() { + if args.types().next().is_some() { let generics = cx.tcx.generics_of(def.did()); let names = get_parameter_names(cx, generics); - let template_params: SmallVec<_> = iter::zip(substs, names) + let template_params: SmallVec<_> = iter::zip(args, names) .filter_map(|(kind, name)| { kind.as_type().map(|ty| { let actual_type = @@ -1295,7 +1289,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( // All function pointers are described as opaque pointers. This could be improved in the future // by describing them as actual function pointers. - let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); + let void_pointer_ty = Ty::new_imm_ptr(tcx, tcx.types.unit); let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); let usize_di_node = type_di_node(cx, tcx.types.usize); let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); @@ -1385,7 +1379,7 @@ fn vcall_visibility_metadata<'ll, 'tcx>( let trait_def_id = trait_ref_self.def_id(); let trait_vis = cx.tcx.visibility(trait_def_id); - let cgus = cx.sess().codegen_units(); + let cgus = cx.sess().codegen_units().as_usize(); let single_cgu = cgus == 1; let lto = cx.sess().lto(); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index ecb0912d328..88040557a9b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -12,7 +12,7 @@ use rustc_middle::{ ty::{ self, layout::{LayoutOf, TyAndLayout}, - AdtDef, GeneratorSubsts, Ty, + AdtDef, GeneratorArgs, Ty, }, }; use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; @@ -199,7 +199,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let enum_type_and_layout = cx.layout_of(enum_type); let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); @@ -667,20 +667,21 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { - let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = + generator_type_and_layout.variants + else { bug!("This function only supports layouts with directly encoded tags.") }; - let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { - &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), + let (generator_def_id, generator_args) = match generator_type_and_layout.ty.kind() { + &ty::Generator(def_id, args, _) => (def_id, args.as_generator()), _ => unreachable!(), }; - let (generator_layout, state_specific_upvar_names) = - cx.tcx.generator_layout_and_saved_local_names(generator_def_id); + let generator_layout = cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(generator_def_id); - let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + let variant_range = generator_args.variant_range(generator_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); let tag_base_type = tag_base_type(cx, generator_type_and_layout); @@ -690,11 +691,11 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_di_node, variant_range .clone() - .map(|variant_index| (variant_index, GeneratorSubsts::variant_name(variant_index))), + .map(|variant_index| (variant_index, GeneratorArgs::variant_name(variant_index))), ); let discriminants: IndexVec<VariantIdx, DiscrResult> = { - let discriminants_iter = generator_substs.discriminants(generator_def_id, cx.tcx); + let discriminants_iter = generator_args.discriminants(generator_def_id, cx.tcx); let mut discriminants: IndexVec<VariantIdx, DiscrResult> = IndexVec::with_capacity(variant_count); for (variant_index, discr) in discriminants_iter { @@ -714,7 +715,6 @@ fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( generator_type_and_layout, generator_type_di_node, generator_layout, - &state_specific_upvar_names, &common_upvar_names, ); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 9e0e847a155..b4beb80ca8b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -6,11 +6,11 @@ use rustc_hir::def::CtorKind; use rustc_index::IndexSlice; use rustc_middle::{ bug, - mir::{GeneratorLayout, GeneratorSavedLocal}, + mir::GeneratorLayout, ty::{ self, layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, - AdtDef, GeneratorSubsts, Ty, VariantDef, + AdtDef, GeneratorArgs, Ty, VariantDef, }, }; use rustc_span::Symbol; @@ -51,7 +51,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let enum_type_and_layout = cx.layout_of(enum_type); @@ -323,10 +323,9 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( generator_type_and_layout: TyAndLayout<'tcx>, generator_type_di_node: &'ll DIType, generator_layout: &GeneratorLayout<'tcx>, - state_specific_upvar_names: &IndexSlice<GeneratorSavedLocal, Option<Symbol>>, - common_upvar_names: &[String], + common_upvar_names: &IndexSlice<FieldIdx, Symbol>, ) -> &'ll DIType { - let variant_name = GeneratorSubsts::variant_name(variant_index); + let variant_name = GeneratorArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( cx.tcx, generator_type_and_layout.ty, @@ -335,8 +334,8 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); - let generator_substs = match generator_type_and_layout.ty.kind() { - ty::Generator(_, substs, _) => substs.as_generator(), + let generator_args = match generator_type_and_layout.ty.kind() { + ty::Generator(_, args, _) => args.as_generator(), _ => unreachable!(), }; @@ -357,7 +356,7 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .map(|field_index| { let generator_saved_local = generator_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; - let field_name_maybe = state_specific_upvar_names[generator_saved_local]; + let field_name_maybe = generator_layout.field_names[generator_saved_local]; let field_name = field_name_maybe .as_ref() .map(|s| Cow::from(s.as_str())) @@ -378,14 +377,15 @@ pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( .collect(); // Fields that are common to all states - let common_fields: SmallVec<_> = generator_substs + let common_fields: SmallVec<_> = generator_args .prefix_tys() + .zip(common_upvar_names) .enumerate() - .map(|(index, upvar_ty)| { + .map(|(index, (upvar_ty, upvar_name))| { build_field_di_node( cx, variant_struct_type_di_node, - &common_upvar_names[index], + upvar_name.as_str(), cx.size_and_align_of(upvar_ty), generator_type_and_layout.fields.offset(index), DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 978141917c6..feac40d8c30 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -57,7 +57,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>( let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) - }; + }; let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); let enum_type_and_layout = cx.layout_of(enum_type); @@ -132,9 +132,9 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( unique_type_id: UniqueTypeId<'tcx>, ) -> DINodeCreationResult<'ll> { let generator_type = unique_type_id.expect_ty(); - let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { + let &ty::Generator(generator_def_id, _, _) = generator_type.kind() else { bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) - }; + }; let containing_scope = get_namespace_for_item(cx, generator_def_id); let generator_type_and_layout = cx.layout_of(generator_type); @@ -155,10 +155,12 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( DIFlags::FlagZero, ), |cx, generator_type_di_node| { - let (generator_layout, state_specific_upvar_names) = - cx.tcx.generator_layout_and_saved_local_names(generator_def_id); + let generator_layout = + cx.tcx.optimized_mir(generator_def_id).generator_layout().unwrap(); - let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = + generator_type_and_layout.variants + else { bug!( "Encountered generator with non-direct-tag layout: {:?}", generator_type_and_layout @@ -173,7 +175,7 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( .indices() .map(|variant_index| { // FIXME: This is problematic because just a number is not a valid identifier. - // GeneratorSubsts::variant_name(variant_index), would be consistent + // GeneratorArgs::variant_name(variant_index), would be consistent // with enums? let variant_name = format!("{}", variant_index.as_usize()).into(); @@ -195,7 +197,6 @@ pub(super) fn build_generator_di_node<'ll, 'tcx>( generator_type_and_layout, generator_type_di_node, generator_layout, - &state_specific_upvar_names, &common_upvar_names, ), source_info, @@ -412,13 +413,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( enum_type_and_layout.size.bits(), enum_type_and_layout.align.abi.bits() as u32, Size::ZERO.bits(), - discr_value.opt_single_val().map(|value| { - // NOTE(eddyb) do *NOT* remove this assert, until - // we pass the full 128-bit value to LLVM, otherwise - // truncation will be silent and remain undetected. - assert_eq!(value as u64 as u128, value); - cx.const_u64(value as u64) - }), + discr_value.opt_single_val().map(|value| cx.const_u128(value)), DIFlags::FlagZero, variant_member_info.variant_struct_type_di_node, ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index aa7ae9355bc..40714a0afe9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -27,7 +27,7 @@ use rustc_hir::def_id::{DefId, DefIdMap}; use rustc_index::IndexVec; use rustc_middle::mir; use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::subst::SubstsRef; +use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt}; use rustc_session::config::{self, DebugInfo}; use rustc_session::Session; @@ -113,7 +113,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"Dwarf Version".as_ptr().cast(), + "Dwarf Version\0".as_ptr().cast(), dwarf_version, ); } else { @@ -121,16 +121,17 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"CodeView".as_ptr().cast(), + "CodeView\0".as_ptr().cast(), 1, ) } // Prevent bitcode readers from deleting the debug info. + let ptr = "Debug Info Version\0".as_ptr(); llvm::LLVMRustAddModuleFlag( self.llmod, llvm::LLVMModFlagBehavior::Warning, - c"Debug Info Version".as_ptr().cast(), + ptr.cast(), llvm::LLVMRustDebugMetadataVersion(), ); } @@ -262,7 +263,7 @@ impl CodegenCx<'_, '_> { pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { let (file, line, col) = match self.sess().source_map().lookup_line(pos) { Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.line_begin_pos(pos); + let line_pos = file.lines(|lines| lines[line]); // Use 1-based indexing. let line = (line + 1) as u32; @@ -331,25 +332,25 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) }; - let mut name = String::new(); + let mut name = String::with_capacity(64); type_names::push_item_name(tcx, def_id, false, &mut name); // Find the enclosing function, in case this is a closure. let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); - // We look up the generics of the enclosing function and truncate the substs + // We look up the generics of the enclosing function and truncate the args // to their length in order to cut off extra stuff that might be in there for // closures or generators. let generics = tcx.generics_of(enclosing_fn_def_id); - let substs = instance.substs.truncate_to(tcx, generics); + let args = instance.args.truncate_to(tcx, generics); type_names::push_generic_params( tcx, - tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), &mut name, ); - let template_parameters = get_template_parameters(self, generics, substs); + let template_parameters = get_template_parameters(self, generics, args); let linkage_name = &mangled_name_of_instance(self, instance).name; // Omit the linkage_name if it is the same as subprogram name. @@ -453,7 +454,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { ty::Array(ct, _) if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => { - cx.tcx.mk_imm_ptr(*ct) + Ty::new_imm_ptr(cx.tcx, *ct) } _ => t, }; @@ -470,16 +471,16 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn get_template_parameters<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, generics: &ty::Generics, - substs: SubstsRef<'tcx>, + args: GenericArgsRef<'tcx>, ) -> &'ll DIArray { - if substs.types().next().is_none() { + if args.types().next().is_none() { return create_DIArray(DIB(cx), &[]); } // Again, only create type information if full debuginfo is enabled let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { let names = get_parameter_names(cx, generics); - iter::zip(substs, names) + iter::zip(args, names) .filter_map(|(kind, name)| { kind.as_type().map(|ty| { let actual_type = @@ -526,7 +527,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { // If the method does *not* belong to a trait, proceed if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( - instance.substs, + instance.args, ty::ParamEnv::reveal_all(), cx.tcx.type_of(impl_def_id), ); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index d5ea48c311b..fa61c7dde18 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -28,7 +28,7 @@ pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DISco .map(|parent| item_namespace(cx, DefId { krate: def_id.krate, index: parent })); let namespace_name_string = { - let mut output = String::new(); + let mut output = String::with_capacity(64); type_names::push_item_name(cx.tcx, def_id, false, &mut output); output }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index 6bcd3e5bf58..7be83638676 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -82,8 +82,8 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>( ty::Foreign(_) => { // Assert that pointers to foreign types really are thin: debug_assert_eq!( - cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), - cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) + cx.size_of(Ty::new_imm_ptr(cx.tcx, pointee_tail_ty)), + cx.size_of(Ty::new_imm_ptr(cx.tcx, cx.tcx.types.u8)) ); None } diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index 44869ced1ae..fced6d504d2 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -81,6 +81,8 @@ pub(crate) struct ErrorCallingDllTool<'a> { #[derive(Diagnostic)] #[diag(codegen_llvm_dlltool_fail_import_library)] pub(crate) struct DlltoolFailImportLibrary<'a> { + pub dlltool_path: Cow<'a, str>, + pub dlltool_args: String, pub stdout: Cow<'a, str>, pub stderr: Cow<'a, str>, } diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4e28034a850..623c08ce86b 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -7,7 +7,7 @@ use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; use crate::value::Value; -use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; +use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::OperandRef; @@ -90,7 +90,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let tcx = self.tcx; let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); - let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { bug!("expected fn item type, found {}", callee_ty); }; @@ -163,7 +163,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::volatile_load | sym::unaligned_volatile_load => { - let tp_ty = substs.type_at(0); + let tp_ty = fn_args.type_at(0); let ptr = args[0].immediate(); let load = if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let llty = ty.llvm_type(self); @@ -298,7 +298,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::raw_eq => { use abi::Abi::*; - let tp_ty = substs.type_at(0); + let tp_ty = fn_args.type_at(0); let layout = self.layout_of(tp_ty).layout; let use_integer_compare = match layout.abi() { Scalar(_) | ScalarPair(_, _) => true, @@ -452,6 +452,8 @@ fn try_intrinsic<'ll>( bx.store(bx.const_i32(0), dest, ret_align); } else if wants_msvc_seh(bx.sess()) { codegen_msvc_try(bx, try_func, data, catch_func, dest); + } else if wants_wasm_eh(bx.sess()) { + codegen_wasm_try(bx, try_func, data, catch_func, dest); } else if bx.sess().target.os == "emscripten" { codegen_emcc_try(bx, try_func, data, catch_func, dest); } else { @@ -610,6 +612,80 @@ fn codegen_msvc_try<'ll>( bx.store(ret, dest, i32_align); } +// WASM's definition of the `rust_try` function. +fn codegen_wasm_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + bx.set_personality_fn(bx.eh_personality()); + + let normal = bx.append_sibling_block("normal"); + let catchswitch = bx.append_sibling_block("catchswitch"); + let catchpad = bx.append_sibling_block("catchpad"); + let caught = bx.append_sibling_block("caught"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + + // We're generating an IR snippet that looks like: + // + // declare i32 @rust_try(%try_func, %data, %catch_func) { + // %slot = alloca i8* + // invoke %try_func(%data) to label %normal unwind label %catchswitch + // + // normal: + // ret i32 0 + // + // catchswitch: + // %cs = catchswitch within none [%catchpad] unwind to caller + // + // catchpad: + // %tok = catchpad within %cs [null] + // %ptr = call @llvm.wasm.get.exception(token %tok) + // %sel = call @llvm.wasm.get.ehselector(token %tok) + // call %catch_func(%data, %ptr) + // catchret from %tok to label %caught + // + // caught: + // ret i32 1 + // } + // + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); + + bx.switch_to_block(normal); + bx.ret(bx.const_i32(0)); + + bx.switch_to_block(catchswitch); + let cs = bx.catch_switch(None, None, &[catchpad]); + + bx.switch_to_block(catchpad); + let null = bx.const_null(bx.type_i8p()); + let funclet = bx.catch_pad(cs, &[null]); + + let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]); + let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]); + + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + bx.switch_to_block(caught); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + // Definition of the standard `try` function for Rust using the GNU-like model // of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` // instructions). @@ -797,23 +873,29 @@ fn get_rust_try_fn<'ll, 'tcx>( // Define the type up front for the signature of the rust_try function. let tcx = cx.tcx; - let i8p = tcx.mk_mut_ptr(tcx.types.i8); + let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8); // `unsafe fn(*mut i8) -> ()` - let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - [i8p], - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); + let try_fn_ty = Ty::new_fn_ptr( + tcx, + ty::Binder::dummy(tcx.mk_fn_sig( + [i8p], + Ty::new_unit(tcx), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ); // `unsafe fn(*mut i8, *mut i8) -> ()` - let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( - [i8p, i8p], - tcx.mk_unit(), - false, - hir::Unsafety::Unsafe, - Abi::Rust, - ))); + let catch_fn_ty = Ty::new_fn_ptr( + tcx, + ty::Binder::dummy(tcx.mk_fn_sig( + [i8p, i8p], + Ty::new_unit(tcx), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( [try_fn_ty, i8p, catch_fn_ty], @@ -1992,6 +2074,53 @@ fn generic_simd_intrinsic<'ll, 'tcx>( simd_neg: Int => neg, Float => fneg; } + // Unary integer intrinsics + if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { + let vec_ty = bx.cx.type_vector( + match *in_elem.kind() { + ty::Int(i) => bx.cx.type_int_from_ty(i), + ty::Uint(i) => bx.cx.type_uint_from_ty(i), + _ => return_error!(InvalidMonomorphization::UnsupportedOperation { + span, + name, + in_ty, + in_elem + }), + }, + in_len as u64, + ); + let intrinsic_name = match name { + sym::simd_bswap => "bswap", + sym::simd_bitreverse => "bitreverse", + sym::simd_ctlz => "ctlz", + sym::simd_cttz => "cttz", + _ => unreachable!(), + }; + let llvm_intrinsic = &format!( + "llvm.{}.v{}i{}", + intrinsic_name, + in_len, + in_elem.int_size_and_signed(bx.tcx()).0.bits(), + ); + + return Ok(if matches!(name, sym::simd_ctlz | sym::simd_cttz) { + let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + bx.call( + fn_ty, + None, + None, + f, + &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], + None, + ) + } else { + let fn_ty = bx.type_func(&[vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + bx.call(fn_ty, None, None, f, &[args[0].immediate()], None) + }); + } + if name == sym::simd_arith_offset { // This also checks that the first operand is a ptr type. let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ff9909c720e..5aab06febe4 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -11,7 +11,6 @@ #![feature(let_chains)] #![feature(never_type)] #![feature(impl_trait_in_assoc_type)] -#![feature(c_str_literals)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] @@ -34,19 +33,20 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::{CodegenResults, CompiledModule}; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::FxIndexMap; use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage}; use rustc_fluent_macro::fluent_messages; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; +use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; use std::any::Any; use std::ffi::CStr; +use std::io::Write; mod back { pub mod archive; @@ -179,7 +179,28 @@ impl WriteBackendMethods for LlvmCodegenBackend { type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { unsafe { - llvm::LLVMRustPrintPassTimings(); + let mut size = 0; + let cstr = llvm::LLVMRustPrintPassTimings(&mut size as *mut usize); + if cstr.is_null() { + println!("failed to get pass timings"); + } else { + let timings = std::slice::from_raw_parts(cstr as *const u8, size); + std::io::stdout().write_all(timings).unwrap(); + libc::free(cstr as *mut _); + } + } + } + fn print_statistics(&self) { + unsafe { + let mut size = 0; + let cstr = llvm::LLVMRustPrintStatistics(&mut size as *mut usize); + if cstr.is_null() { + println!("failed to get pass stats"); + } else { + let stats = std::slice::from_raw_parts(cstr as *const u8, size); + std::io::stdout().write_all(stats).unwrap(); + libc::free(cstr as *mut _); + } } } fn run_link( @@ -263,10 +284,10 @@ impl CodegenBackend for LlvmCodegenBackend { |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) } - fn print(&self, req: PrintRequest, sess: &Session) { - match req { - PrintRequest::RelocationModels => { - println!("Available relocation models:"); + fn print(&self, req: &PrintRequest, out: &mut dyn PrintBackendInfo, sess: &Session) { + match req.kind { + PrintKind::RelocationModels => { + writeln!(out, "Available relocation models:"); for name in &[ "static", "pic", @@ -277,26 +298,27 @@ impl CodegenBackend for LlvmCodegenBackend { "ropi-rwpi", "default", ] { - println!(" {}", name); + writeln!(out, " {}", name); } - println!(); + writeln!(out); } - PrintRequest::CodeModels => { - println!("Available code models:"); + PrintKind::CodeModels => { + writeln!(out, "Available code models:"); for name in &["tiny", "small", "kernel", "medium", "large"] { - println!(" {}", name); + writeln!(out, " {}", name); } - println!(); + writeln!(out); } - PrintRequest::TlsModels => { - println!("Available TLS models:"); + PrintKind::TlsModels => { + writeln!(out, "Available TLS models:"); for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { - println!(" {}", name); + writeln!(out, " {}", name); } - println!(); + writeln!(out); } - PrintRequest::StackProtectorStrategies => { - println!( + PrintKind::StackProtectorStrategies => { + writeln!( + out, r#"Available stack protector strategies: all Generate stack canaries in all functions. @@ -320,7 +342,7 @@ impl CodegenBackend for LlvmCodegenBackend { "# ); } - req => llvm_util::print(req, sess), + _other => llvm_util::print(req, out, sess), } } @@ -356,7 +378,7 @@ impl CodegenBackend for LlvmCodegenBackend { ongoing_codegen: Box<dyn Any>, sess: &Session, outputs: &OutputFilenames, - ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { + ) -> Result<(CodegenResults, FxIndexMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { let (codegen_results, work_products) = ongoing_codegen .downcast::<rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6ef3418cc5f..5fe6407ac4f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,7 +1,7 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] -use rustc_codegen_ssa::coverageinfo::map as coverage_map; +use crate::coverageinfo::map_data as coverage_map; use super::debuginfo::{ DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, @@ -585,6 +585,16 @@ pub enum ThreadLocalMode { LocalExec, } +/// LLVMRustTailCallKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum TailCallKind { + None, + Tail, + MustTail, + NoTail, +} + /// LLVMRustChecksumKind #[derive(Copy, Clone)] #[repr(C)] @@ -1071,6 +1081,7 @@ extern "C" { // Operations on other types pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub fn LLVMTokenTypeInContext(C: &Context) -> &Type; pub fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values @@ -1144,7 +1155,7 @@ extern "C" { pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; // Constant expressions - pub fn LLVMRustConstInBoundsGEP2<'a>( + pub fn LLVMConstInBoundsGEP2<'a>( ty: &'a Type, ConstantVal: &'a Value, ConstantIndices: *const &'a Value, @@ -1195,6 +1206,7 @@ extern "C" { NameLen: size_t, ) -> Option<&Value>; pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub fn LLVMRustSetTailCallKind(CallInst: &Value, TKC: TailCallKind); // Operations on attributes pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; @@ -1301,7 +1313,7 @@ extern "C" { NumArgs: c_uint, Then: &'a BasicBlock, Catch: &'a BasicBlock, - OpBundles: *const Option<&OperandBundleDef<'a>>, + OpBundles: *const &OperandBundleDef<'a>, NumOpBundles: c_uint, Name: *const c_char, ) -> &'a Value; @@ -1673,7 +1685,7 @@ extern "C" { Fn: &'a Value, Args: *const &'a Value, NumArgs: c_uint, - OpBundles: *const Option<&OperandBundleDef<'a>>, + OpBundles: *const &OperandBundleDef<'a>, NumOpBundles: c_uint, ) -> &'a Value; pub fn LLVMRustBuildMemCpy<'a>( @@ -1856,7 +1868,10 @@ extern "C" { pub fn LLVMRustGetLastError() -> *const c_char; /// Print the pass timings since static dtors aren't picking them up. - pub fn LLVMRustPrintPassTimings(); + pub fn LLVMRustPrintPassTimings(size: *const size_t) -> *const c_char; + + /// Print the statistics since static dtors aren't picking them up. + pub fn LLVMRustPrintStatistics(size: *const size_t) -> *const c_char; pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; @@ -1904,7 +1919,6 @@ extern "C" { ); pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; - pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] @@ -2269,7 +2283,12 @@ extern "C" { pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; - pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine, cpu: *const c_char); + pub fn LLVMRustPrintTargetCPUs( + T: &TargetMachine, + cpu: *const c_char, + print: unsafe extern "C" fn(out: *mut c_void, string: *const c_char, len: usize), + out: *mut c_void, + ); pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; pub fn LLVMRustGetTargetFeature( T: &TargetMachine, @@ -2512,6 +2531,7 @@ extern "C" { remark_all_passes: bool, remark_passes: *const *const c_char, remark_passes_len: usize, + remark_file: *const c_char, ); #[allow(improper_ctypes)] diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 03be0654b50..6ae738174bf 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -8,16 +8,17 @@ use libc::c_int; use rustc_codegen_ssa::target_features::{ supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, }; +use rustc_codegen_ssa::traits::PrintBackendInfo; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::small_c_str::SmallCStr; use rustc_fs_util::path_to_c_string; use rustc_middle::bug; -use rustc_session::config::PrintRequest; +use rustc_session::config::{PrintKind, PrintRequest}; use rustc_session::Session; use rustc_span::symbol::Symbol; use rustc_target::spec::{MergeFunctions, PanicStrategy}; -use std::ffi::{CStr, CString}; +use std::ffi::{c_char, c_void, CStr, CString}; use std::path::Path; use std::ptr; use std::slice; @@ -110,6 +111,10 @@ unsafe fn configure_llvm(sess: &Session) { // Use non-zero `import-instr-limit` multiplier for cold callsites. add("-import-cold-multiplier=0.1", false); + if sess.print_llvm_stats() { + add("-stats", false); + } + for arg in sess_args { add(&(*arg), true); } @@ -350,7 +355,7 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { ret } -fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { +fn print_target_features(out: &mut dyn PrintBackendInfo, sess: &Session, tm: &llvm::TargetMachine) { let mut llvm_target_features = llvm_target_features(tm); let mut known_llvm_target_features = FxHashSet::<&'static str>::default(); let mut rustc_target_features = supported_target_features(sess) @@ -383,36 +388,48 @@ fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { .max() .unwrap_or(0); - println!("Features supported by rustc for this target:"); + writeln!(out, "Features supported by rustc for this target:"); for (feature, desc) in &rustc_target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + writeln!(out, " {1:0$} - {2}.", max_feature_len, feature, desc); } - println!("\nCode-generation features supported by LLVM for this target:"); + writeln!(out, "\nCode-generation features supported by LLVM for this target:"); for (feature, desc) in &llvm_target_features { - println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + writeln!(out, " {1:0$} - {2}.", max_feature_len, feature, desc); } if llvm_target_features.is_empty() { - println!(" Target features listing is not supported by this LLVM version."); + writeln!(out, " Target features listing is not supported by this LLVM version."); } - println!("\nUse +feature to enable a feature, or -feature to disable it."); - println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); - println!("Code-generation features cannot be used in cfg or #[target_feature],"); - println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); + writeln!(out, "\nUse +feature to enable a feature, or -feature to disable it."); + writeln!(out, "For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); + writeln!(out, "Code-generation features cannot be used in cfg or #[target_feature],"); + writeln!(out, "and may be renamed or removed in a future version of LLVM or rustc.\n"); } -pub(crate) fn print(req: PrintRequest, sess: &Session) { +pub(crate) fn print(req: &PrintRequest, mut out: &mut dyn PrintBackendInfo, sess: &Session) { require_inited(); let tm = create_informational_target_machine(sess); - match req { - PrintRequest::TargetCPUs => { + match req.kind { + PrintKind::TargetCPUs => { // SAFETY generate a C compatible string from a byte slice to pass // the target CPU name into LLVM, the lifetime of the reference is // at least as long as the C function let cpu_cstring = CString::new(handle_native(sess.target.cpu.as_ref())) .unwrap_or_else(|e| bug!("failed to convert to cstring: {}", e)); - unsafe { llvm::LLVMRustPrintTargetCPUs(tm, cpu_cstring.as_ptr()) }; + unsafe extern "C" fn callback(out: *mut c_void, string: *const c_char, len: usize) { + let out = &mut *(out as *mut &mut dyn PrintBackendInfo); + let bytes = slice::from_raw_parts(string as *const u8, len); + write!(out, "{}", String::from_utf8_lossy(bytes)); + } + unsafe { + llvm::LLVMRustPrintTargetCPUs( + tm, + cpu_cstring.as_ptr(), + callback, + &mut out as *mut &mut dyn PrintBackendInfo as *mut c_void, + ); + } } - PrintRequest::TargetFeatures => print_target_features(sess, tm), + PrintKind::TargetFeatures => print_target_features(out, sess, tm), _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), } } @@ -490,8 +507,6 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str .features .split(',') .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) - // Drop +atomics-32 feature introduced in LLVM 15. - .filter(|v| *v != "+atomics-32" || get_version() >= (15, 0, 0)) .map(String::from), ); diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index c24854b27a0..e8cda626f54 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -48,7 +48,7 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { visibility: Visibility, symbol_name: &str, ) { - assert!(!instance.substs.has_infer()); + assert!(!instance.args.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index d3fad5699c8..7e672a8dc33 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -52,6 +52,10 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } } + pub(crate) fn type_token(&self) -> &'ll Type { + unsafe { llvm::LLVMTokenTypeInContext(self.llcx) } + } + pub(crate) fn type_metadata(&self) -> &'ll Type { unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) } } @@ -288,6 +292,9 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } + fn scalar_copy_backend_type(&self, layout: TyAndLayout<'tcx>) -> Option<Self::Type> { + layout.scalar_copy_llvm_type(self) + } } impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index a493c9c0548..2dbd467cc84 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -6,6 +6,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, Ty, TypeVisitableExt}; +use rustc_target::abi::HasDataLayout; use rustc_target::abi::{Abi, Align, FieldsShape}; use rustc_target::abi::{Int, Pointer, F32, F64}; use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; @@ -56,7 +57,7 @@ fn uncached_llvm_type<'a, 'tcx>( if let (&ty::Generator(_, _, _), &Variants::Single { index }) = (layout.ty.kind(), &layout.variants) { - write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap(); + write!(&mut name, "::{}", ty::GeneratorArgs::variant_name(index)).unwrap(); } Some(name) } @@ -192,6 +193,7 @@ pub trait LayoutLlvmExt<'tcx> { ) -> &'a Type; fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo>; + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>; } impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { @@ -334,13 +336,14 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { } // only wide pointer boxes are handled as pointers // thin pointer boxes with scalar allocators are handled by the general logic below - ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { - let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); + ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_zst() => { + let ptr_ty = Ty::new_mut_ptr(cx.tcx, self.ty.boxed_ty()); return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); } // `dyn* Trait` has the same ABI as `*mut dyn Trait` ty::Dynamic(bounds, region, ty::DynStar) => { - let ptr_ty = cx.tcx.mk_mut_ptr(cx.tcx.mk_dynamic(bounds, region, ty::Dyn)); + let ptr_ty = + Ty::new_mut_ptr(cx.tcx, Ty::new_dynamic(cx.tcx, bounds, region, ty::Dyn)); return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); } _ => {} @@ -414,4 +417,35 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); result } + + fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> { + debug_assert!(self.is_sized()); + + // FIXME: this is a fairly arbitrary choice, but 128 bits on WASM + // (matching the 128-bit SIMD types proposal) and 256 bits on x64 + // (like AVX2 registers) seems at least like a tolerable starting point. + let threshold = cx.data_layout().pointer_size * 4; + if self.layout.size() > threshold { + return None; + } + + // Vectors, even for non-power-of-two sizes, have the same layout as + // arrays but don't count as aggregate types + if let FieldsShape::Array { count, .. } = self.layout.fields() + && let element = self.field(cx, 0) + && element.ty.is_integral() + { + // `cx.type_ix(bits)` is tempting here, but while that works great + // for things that *stay* as memory-to-memory copies, it also ends + // up suppressing vectorization as it introduces shifts when it + // extracts all the individual values. + + let ety = element.llvm_type(cx); + return Some(cx.type_vector(ety, *count)); + } + + // FIXME: The above only handled integer arrays; surely more things + // would also be possible. Be careful about provenance, though! + None + } } diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index b19398e68c2..8800caa71d6 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -73,7 +73,7 @@ fn emit_ptr_va_arg<'ll, 'tcx>( let layout = bx.cx.layout_of(target_ty); let (llty, size, align) = if indirect { ( - bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), + bx.cx.layout_of(Ty::new_imm_ptr(bx.cx.tcx, target_ty)).llvm_type(bx.cx), bx.cx.data_layout().pointer_size, bx.cx.data_layout().pointer_align, ) |