diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
28 files changed, 701 insertions, 1013 deletions
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 2fe5ed32daa..8a1ee48c43c 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -415,7 +415,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { instance: Option<ty::Instance<'tcx>>, ) { let mut func_attrs = SmallVec::<[_; 3]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); } if !self.can_unwind { @@ -532,7 +532,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { let mut func_attrs = SmallVec::<[_; 2]>::new(); - if self.ret.layout.abi.is_uninhabited() { + if self.ret.layout.is_uninhabited() { func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); } if !self.can_unwind { diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 89f5305840b..a2e6c7eb856 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -7,6 +7,7 @@ use rustc_middle::bug; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; +use crate::common::AsCCharPtr; use crate::llvm::{self, Context, False, Module, True, Type}; use crate::{ModuleLlvm, attributes, debuginfo}; @@ -76,21 +77,15 @@ pub(crate) unsafe fn codegen( unsafe { // __rust_alloc_error_handler_should_panic let name = OomStrategy::SYMBOL; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - llvm::LLVMRustSetVisibility( - ll_g, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let val = tcx.sess.opts.unstable_opts.oom.should_panic(); let llval = llvm::LLVMConstInt(i8, val as u64, False); llvm::LLVMSetInitializer(ll_g, llval); let name = NO_ALLOC_SHIM_IS_UNSTABLE; - let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); - llvm::LLVMRustSetVisibility( - ll_g, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_c_char_ptr(), name.len(), i8); + llvm::set_visibility(ll_g, llvm::Visibility::from_generic(tcx.sess.default_visibility())); let llval = llvm::LLVMConstInt(i8, 0, False); llvm::LLVMSetInitializer(ll_g, llval); } @@ -121,7 +116,7 @@ fn create_wrapper_function( ); let llfn = llvm::LLVMRustGetOrInsertFunction( llmod, - from_name.as_ptr().cast(), + from_name.as_c_char_ptr(), from_name.len(), ty, ); @@ -134,10 +129,7 @@ fn create_wrapper_function( None }; - llvm::LLVMRustSetVisibility( - llfn, - llvm::Visibility::from_generic(tcx.sess.default_visibility()), - ); + llvm::set_visibility(llfn, llvm::Visibility::from_generic(tcx.sess.default_visibility())); if tcx.sess.must_emit_unwind_tables() { let uwtable = @@ -146,12 +138,12 @@ fn create_wrapper_function( } let callee = - llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_ptr().cast(), to_name.len(), ty); + llvm::LLVMRustGetOrInsertFunction(llmod, to_name.as_c_char_ptr(), to_name.len(), ty); if let Some(no_return) = no_return { // -> ! DIFlagNoReturn attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); } - llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + llvm::set_visibility(callee, llvm::Visibility::Hidden); let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr()); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index d1d7d0cf4ce..3c30822a2e2 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -15,7 +15,7 @@ use smallvec::SmallVec; use tracing::debug; use crate::builder::Builder; -use crate::common::Funclet; +use crate::common::{AsCCharPtr, Funclet}; use crate::context::CodegenCx; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; @@ -420,7 +420,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { unsafe { llvm::LLVMAppendModuleInlineAsm( self.llmod, - template_str.as_ptr().cast(), + template_str.as_c_char_ptr(), template_str.len(), ); } @@ -458,14 +458,14 @@ pub(crate) fn inline_asm_call<'ll>( let fty = bx.cx.type_func(&argtys, output); unsafe { // Ask LLVM to verify that the constraints are well-formed. - let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_c_char_ptr(), cons.len()); debug!("constraint verification result: {:?}", constraints_ok); if constraints_ok { let v = llvm::LLVMRustInlineAsm( fty, - asm.as_ptr().cast(), + asm.as_c_char_ptr(), asm.len(), - cons.as_ptr().cast(), + cons.as_c_char_ptr(), cons.len(), volatile, alignstack, diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 1f7a923dd2c..02149e176ea 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -502,9 +502,9 @@ fn thin_lto( // upstream... let data = llvm::LLVMRustCreateThinLTOData( thin_modules.as_ptr(), - thin_modules.len() as u32, + thin_modules.len(), symbols_below_threshold.as_ptr(), - symbols_below_threshold.len() as u32, + symbols_below_threshold.len(), ) .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; @@ -569,7 +569,7 @@ fn thin_lto( info!(" - {}: re-compiled", module_name); opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { - shared: shared.clone(), + shared: Arc::clone(&shared), idx: module_index, })); } @@ -601,23 +601,9 @@ pub(crate) fn run_pass_manager( // This code is based off the code found in llvm's LTO code generator: // llvm/lib/LTO/LTOCodeGenerator.cpp debug!("running the pass manager"); - unsafe { - if !llvm::LLVMRustHasModuleFlag( - module.module_llvm.llmod(), - "LTOPostLink".as_ptr().cast(), - 11, - ) { - llvm::LLVMRustAddModuleFlagU32( - module.module_llvm.llmod(), - llvm::LLVMModFlagBehavior::Error, - c"LTOPostLink".as_ptr(), - 1, - ); - } - let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; - let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); - write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?; - } + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + unsafe { write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage) }?; debug!("lto done"); Ok(()) } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index afdd2b581b8..e68ba96f14f 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -34,6 +34,7 @@ use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback, }; +use crate::common::AsCCharPtr; use crate::errors::{ CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, UnknownCompression, WithLlvmError, WriteBytecode, @@ -596,9 +597,9 @@ pub(crate) unsafe fn llvm_optimize( llvm_selfprofiler, selfprofile_before_pass_callback, selfprofile_after_pass_callback, - extra_passes.as_ptr().cast(), + extra_passes.as_c_char_ptr(), extra_passes.len(), - llvm_plugins.as_ptr().cast(), + llvm_plugins.as_c_char_ptr(), llvm_plugins.len(), ) }; @@ -1042,8 +1043,8 @@ unsafe fn embed_bitcode( llvm::LLVMSetInitializer(llglobal, llconst); let section = bitcode_section_name(cgcx); - llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetSection(llglobal, section.as_c_char_ptr()); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); llvm::LLVMSetGlobalConstant(llglobal, llvm::True); let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); @@ -1061,14 +1062,14 @@ unsafe fn embed_bitcode( c".llvmcmd" }; llvm::LLVMSetSection(llglobal, section.as_ptr()); - llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); } else { // We need custom section flags, so emit module-level inline assembly. let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); - llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + llvm::LLVMAppendModuleInlineAsm(llmod, asm.as_c_char_ptr(), asm.len()); } } } @@ -1096,7 +1097,7 @@ fn create_msvc_imps( let ptr_ty = Type::ptr_llcx(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { - llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage + llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && llvm::LLVMIsDeclaration(val) == 0 }) .filter_map(|val| { @@ -1115,7 +1116,7 @@ fn create_msvc_imps( for (imp_name, val) in globals { let imp = llvm::LLVMAddGlobal(llmod, ptr_ty, imp_name.as_ptr()); llvm::LLVMSetInitializer(imp, val); - llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); + llvm::set_linkage(imp, llvm::Linkage::ExternalLinkage); } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index dbf5298d64b..8702532c36e 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1165,39 +1165,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); } - fn instrprof_increment( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - num_counters: &'ll Value, - index: &'ll Value, - ) { - debug!( - "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, num_counters, index - ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, num_counters, index]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } - } - fn call( &mut self, llty: &'ll Type, @@ -1667,6 +1634,18 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi_bundle } + /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation. + #[instrument(level = "debug", skip(self))] + pub(crate) fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]); + } + /// Emits a call to `llvm.instrprof.mcdc.parameters`. /// /// This doesn't produce any code directly, but is used as input by @@ -1676,40 +1655,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { /// /// [`CodeGenPGO::emitMCDCParameters`]: /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_parameters( &mut self, fn_name: &'ll Value, hash: &'ll Value, bitmap_bits: &'ll Value, ) { - debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits); - assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], - self.cx.type_void(), - ); - let args = &[fn_name, hash, bitmap_bits]; - let args = self.check_call("call", llty, llfn, args); - - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_tvbitmap_update( &mut self, fn_name: &'ll Value, @@ -1717,39 +1677,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { bitmap_index: &'ll Value, mcdc_temp: &'ll Value, ) { - debug!( - "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})", - fn_name, hash, bitmap_index, mcdc_temp - ); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" ); - - let llfn = - unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; - let llty = self.cx.type_func( - &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()], - self.cx.type_void(), - ); let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - let args = self.check_call("call", llty, llfn, args); - unsafe { - let _ = llvm::LLVMRustBuildCall( - self.llbuilder, - llty, - llfn, - args.as_ptr() as *const &llvm::Value, - args.len() as c_uint, - [].as_ptr(), - 0 as c_uint, - ); - } + self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args); + } + + #[instrument(level = "debug", skip(self))] + pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); } + #[instrument(level = "debug", skip(self))] pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp); assert!( crate::llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later" diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 206a7069792..25037b97375 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -95,11 +95,10 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // whether we are sharing generics or not. The important thing here is // that the visibility we apply to the declaration is the same one that // has been applied to the definition (wherever that definition may be). - unsafe { - llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); - let is_generic = - instance.args.non_erasable_generics(tcx, instance.def_id()).next().is_some(); + llvm::set_linkage(llfn, llvm::Linkage::ExternalLinkage); + unsafe { + let is_generic = instance.args.non_erasable_generics().next().is_some(); let is_hidden = if is_generic { // This is a monomorphization of a generic function. @@ -136,7 +135,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t || !cx.tcx.is_reachable_non_generic(instance_def_id)) }; if is_hidden { - llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + llvm::set_visibility(llfn, llvm::Visibility::Hidden); } // MinGW: For backward compatibility we rely on the linker to decide whether it diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 0ced37b53a8..29adc616ee2 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -219,8 +219,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMSetInitializer(g, sc); llvm::LLVMSetGlobalConstant(g, True); llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); } + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); (s.to_owned(), g) }) .1; @@ -392,3 +392,21 @@ pub(crate) fn get_dllimport<'tcx>( tcx.native_library(id) .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name)) } + +/// Extension trait for explicit casts to `*const c_char`. +pub(crate) trait AsCCharPtr { + /// Equivalent to `self.as_ptr().cast()`, but only casts to `*const c_char`. + fn as_c_char_ptr(&self) -> *const c_char; +} + +impl AsCCharPtr for str { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} + +impl AsCCharPtr for [u8] { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 33a85adeb87..21d996ef460 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -19,7 +19,7 @@ use rustc_target::abi::{ }; use tracing::{debug, instrument, trace}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::errors::{ InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined, }; @@ -172,29 +172,27 @@ fn check_and_apply_linkage<'ll, 'tcx>( if let Some(linkage) = attrs.import_linkage { debug!("get_static: sym={} linkage={:?}", sym, linkage); - unsafe { - // Declare a symbol `foo` with the desired linkage. - let g1 = cx.declare_global(sym, cx.type_i8()); - llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); - - // Declare an internal global `extern_with_linkage_foo` which - // is initialized with the address of `foo`. If `foo` is - // discarded during linking (for example, if `foo` has weak - // linkage and there are no definitions), then - // `extern_with_linkage_foo` will instead be initialized to - // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); - let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { - cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { - span: cx.tcx.def_span(def_id), - symbol_name: sym, - }) - }); - llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); - llvm::LLVMSetInitializer(g2, g1); - g2 - } + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, cx.type_i8()); + llvm::set_linkage(g1, base::linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { + cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { + span: cx.tcx.def_span(def_id), + symbol_name: sym, + }) + }); + llvm::set_linkage(g2, llvm::Linkage::InternalLinkage); + unsafe { llvm::LLVMSetInitializer(g2, g1) }; + g2 } else if cx.tcx.sess.target.arch == "x86" && let Some(dllimport) = crate::common::get_dllimport(cx.tcx, def_id, sym) { @@ -224,23 +222,21 @@ impl<'ll> CodegenCx<'ll, '_> { align: Align, kind: Option<&str>, ) -> &'ll Value { - unsafe { - let gv = match kind { - Some(kind) if !self.tcx.sess.fewer_names() => { - let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { - bug!("symbol `{}` is already defined", name); - }); - llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); - gv - } - _ => self.define_private_global(self.val_ty(cv)), - }; - llvm::LLVMSetInitializer(gv, cv); - set_global_alignment(self, gv, align); - llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); - gv - } + let gv = match kind { + Some(kind) if !self.tcx.sess.fewer_names() => { + let name = self.generate_local_symbol_name(kind); + let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); + gv + } + _ => self.define_private_global(self.val_ty(cv)), + }; + unsafe { llvm::LLVMSetInitializer(gv, cv) }; + set_global_alignment(self, gv, align); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv } #[instrument(level = "debug", skip(self))] @@ -292,9 +288,7 @@ impl<'ll> CodegenCx<'ll, '_> { let g = self.declare_global(sym, llty); if !self.tcx.is_reachable_non_generic(def_id) { - unsafe { - llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); - } + llvm::set_visibility(g, llvm::Visibility::Hidden); } g @@ -312,7 +306,7 @@ impl<'ll> CodegenCx<'ll, '_> { llvm::set_thread_local_mode(g, self.tls_model); } - let dso_local = unsafe { self.should_assume_dso_local(g, true) }; + let dso_local = self.should_assume_dso_local(g, true); if dso_local { unsafe { llvm::LLVMRustSetDSOLocal(g, true); @@ -401,18 +395,18 @@ impl<'ll> CodegenCx<'ll, '_> { let name = llvm::get_value_name(g).to_vec(); llvm::set_value_name(g, b""); - let linkage = llvm::LLVMRustGetLinkage(g); - let visibility = llvm::LLVMRustGetVisibility(g); + let linkage = llvm::get_linkage(g); + let visibility = llvm::get_visibility(g); let new_g = llvm::LLVMRustGetOrInsertGlobal( self.llmod, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), val_llty, ); - llvm::LLVMRustSetLinkage(new_g, linkage); - llvm::LLVMRustSetVisibility(new_g, visibility); + llvm::set_linkage(new_g, linkage); + llvm::set_visibility(new_g, visibility); // The old global has had its name removed but is returned by // get_static since it is in the instance cache. Provide an @@ -457,7 +451,7 @@ impl<'ll> CodegenCx<'ll, '_> { if let Some(section) = attrs.link_section { let section = llvm::LLVMMDStringInContext2( self.llcx, - section.as_str().as_ptr().cast(), + section.as_str().as_c_char_ptr(), section.as_str().len(), ); assert!(alloc.provenance().ptrs().is_empty()); @@ -468,7 +462,7 @@ impl<'ll> CodegenCx<'ll, '_> { let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); let alloc = - llvm::LLVMMDStringInContext2(self.llcx, bytes.as_ptr().cast(), bytes.len()); + llvm::LLVMMDStringInContext2(self.llcx, bytes.as_c_char_ptr(), bytes.len()); let data = [section, alloc]; let meta = llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()); let val = llvm::LLVMMetadataAsValue(self.llcx, meta); diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 2f830d6f941..f33da42d63e 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -29,6 +29,7 @@ use smallvec::SmallVec; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; +use crate::common::AsCCharPtr; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{Metadata, MetadataType}; use crate::type_::Type; @@ -80,6 +81,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> { pub isize_ty: &'ll Type, + /// Extra codegen state needed when coverage instrumentation is enabled. pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>, pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>, @@ -208,133 +210,111 @@ pub(crate) unsafe fn create_module<'ll>( // If skipping the PLT is enabled, we need to add some module metadata // to ensure intrinsic calls don't use it. if !sess.needs_plt() { - let avoid_plt = c"RtLibUseGOT".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "RtLibUseGOT", 1); } // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.) if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() { - let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - canonical_jump_tables, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "CFI Canonical Jump Tables", + 1, + ); } // If we're normalizing integers with CFI, ensure LLVM generated functions do the same. // See https://github.com/llvm/llvm-project/pull/104826 if sess.is_sanitizer_cfi_normalize_integers_enabled() { - let cfi_normalize_integers = c"cfi-normalize-integers".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - cfi_normalize_integers, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cfi-normalize-integers", + 1, + ); } // Enable LTO unit splitting if specified or if CFI is enabled. (See // https://reviews.llvm.org/D53891.) if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() { - let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - enable_split_lto_unit, - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "EnableSplitLTOUnit", + 1, + ); } // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.) if sess.is_sanitizer_kcfi_enabled() { - let kcfi = c"kcfi".as_ptr(); - unsafe { - llvm::LLVMRustAddModuleFlagU32(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1); - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Override, "kcfi", 1); // Add "kcfi-offset" module flag with -Z patchable-function-entry (See // https://reviews.llvm.org/D141172). let pfe = PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry); if pfe.prefix() > 0 { - let kcfi_offset = c"kcfi-offset".as_ptr().cast(); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - kcfi_offset, - pfe.prefix().into(), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "kcfi-offset", + pfe.prefix().into(), + ); } } // Control Flow Guard is currently only supported by the MSVC linker on Windows. if sess.target.is_like_msvc { - unsafe { - match sess.opts.cg.control_flow_guard { - CFGuard::Disabled => {} - CFGuard::NoChecks => { - // Set `cfguard=1` module flag to emit metadata only. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 1, - ) - } - CFGuard::Checks => { - // Set `cfguard=2` module flag to emit metadata and checks. - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"cfguard".as_ptr() as *const _, - 2, - ) - } + match sess.opts.cg.control_flow_guard { + CFGuard::Disabled => {} + CFGuard::NoChecks => { + // Set `cfguard=1` module flag to emit metadata only. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 1, + ); + } + CFGuard::Checks => { + // Set `cfguard=2` module flag to emit metadata and checks. + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "cfguard", + 2, + ); } } } if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { if sess.target.arch == "aarch64" { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"branch-target-enforcement".as_ptr(), - bti.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address".as_ptr(), - pac_ret.is_some().into(), - ); - let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-all".as_ptr(), - pac_opts.leaf.into(), - ); - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Min, - c"sign-return-address-with-bkey".as_ptr(), - u32::from(pac_opts.key == PAuthKey::B), - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "branch-target-enforcement", + bti.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address", + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-all", + pac_opts.leaf.into(), + ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "sign-return-address-with-bkey", + u32::from(pac_opts.key == PAuthKey::B), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -345,59 +325,46 @@ pub(crate) unsafe fn create_module<'ll>( // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-branch".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-branch", + 1, + ); } if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Override, - c"cf-protection-return".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "cf-protection-return", + 1, + ); } if sess.opts.unstable_opts.virtual_function_elimination { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"Virtual Function Elim".as_ptr(), - 1, - ); - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Virtual Function Elim", + 1, + ); } // Set module flag to enable Windows EHCont Guard (/guard:ehcont). if sess.opts.unstable_opts.ehcont_guard { - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Warning, - c"ehcontguard".as_ptr() as *const _, - 1, - ) - } + llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "ehcontguard", 1); } match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => {} - FunctionReturn::ThunkExtern => unsafe { - llvm::LLVMRustAddModuleFlagU32( + FunctionReturn::ThunkExtern => { + llvm::add_module_flag_u32( llmod, - llvm::LLVMModFlagBehavior::Override, - c"function_return_thunk_extern".as_ptr(), + llvm::ModuleFlagMergeBehavior::Override, + "function_return_thunk_extern", 1, - ) - }, + ); + } } match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) @@ -405,15 +372,12 @@ pub(crate) unsafe fn create_module<'ll>( // Set up the small-data optimization limit for architectures that use // an LLVM module flag to control this. (Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => { - let flag = SmallCStr::new(flag.as_ref()); - unsafe { - llvm::LLVMRustAddModuleFlagU32( - llmod, - llvm::LLVMModFlagBehavior::Error, - flag.as_c_str().as_ptr(), - threshold as u32, - ) - } + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + &flag, + threshold as u32, + ); } _ => (), }; @@ -428,7 +392,7 @@ pub(crate) unsafe fn create_module<'ll>( let name_metadata = unsafe { llvm::LLVMMDStringInContext2( llcx, - rustc_producer.as_ptr().cast(), + rustc_producer.as_c_char_ptr(), rustc_producer.as_bytes().len(), ) }; @@ -447,33 +411,29 @@ pub(crate) unsafe fn create_module<'ll>( // If llvm_abiname is empty, emit nothing. let llvm_abiname = &sess.target.options.llvm_abiname; if matches!(sess.target.arch.as_ref(), "riscv32" | "riscv64") && !llvm_abiname.is_empty() { - unsafe { - llvm::LLVMRustAddModuleFlagString( - llmod, - llvm::LLVMModFlagBehavior::Error, - c"target-abi".as_ptr(), - llvm_abiname.as_ptr().cast(), - llvm_abiname.len(), - ); - } + llvm::add_module_flag_str( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "target-abi", + llvm_abiname, + ); } // Add module flags specified via -Z llvm_module_flag - for (key, value, behavior) in &sess.opts.unstable_opts.llvm_module_flag { - let key = format!("{key}\0"); - let behavior = match behavior.as_str() { - "error" => llvm::LLVMModFlagBehavior::Error, - "warning" => llvm::LLVMModFlagBehavior::Warning, - "require" => llvm::LLVMModFlagBehavior::Require, - "override" => llvm::LLVMModFlagBehavior::Override, - "append" => llvm::LLVMModFlagBehavior::Append, - "appendunique" => llvm::LLVMModFlagBehavior::AppendUnique, - "max" => llvm::LLVMModFlagBehavior::Max, - "min" => llvm::LLVMModFlagBehavior::Min, + for (key, value, merge_behavior) in &sess.opts.unstable_opts.llvm_module_flag { + let merge_behavior = match merge_behavior.as_str() { + "error" => llvm::ModuleFlagMergeBehavior::Error, + "warning" => llvm::ModuleFlagMergeBehavior::Warning, + "require" => llvm::ModuleFlagMergeBehavior::Require, + "override" => llvm::ModuleFlagMergeBehavior::Override, + "append" => llvm::ModuleFlagMergeBehavior::Append, + "appendunique" => llvm::ModuleFlagMergeBehavior::AppendUnique, + "max" => llvm::ModuleFlagMergeBehavior::Max, + "min" => llvm::ModuleFlagMergeBehavior::Min, // We already checked this during option parsing _ => unreachable!(), }; - unsafe { llvm::LLVMRustAddModuleFlagU32(llmod, behavior, key.as_ptr().cast(), *value) } + llvm::add_module_flag_u32(llmod, merge_behavior, key, *value); } llmod @@ -592,11 +552,10 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { &self.statics_to_rauw } + /// Extra state that is only available when coverage instrumentation is enabled. #[inline] - pub(crate) fn coverage_context( - &self, - ) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { - self.coverage_cx.as_ref() + pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> { + self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled") } pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { @@ -605,7 +564,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { unsafe { let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); llvm::LLVMSetInitializer(g, array); - llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr()); } } @@ -1099,6 +1058,10 @@ impl<'ll> CodegenCx<'ll, '_> { if self.sess().instrument_coverage() { ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); + if crate::llvm_util::get_version() >= (19, 0, 0) { + ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); + ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); + } } ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index 90f7dd733ca..feac97f3e2b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,11 +1,9 @@ -use rustc_middle::mir::coverage::{ - ConditionInfo, CounterId, CovTerm, DecisionInfo, ExpressionId, MappingKind, SourceRegion, -}; +use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion}; /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum CounterKind { +pub(crate) enum CounterKind { Zero = 0, CounterValueReference = 1, Expression = 2, @@ -25,9 +23,9 @@ pub enum CounterKind { /// Must match the layout of `LLVMRustCounter`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct Counter { +pub(crate) struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. - pub kind: CounterKind, + pub(crate) kind: CounterKind, id: u32, } @@ -36,7 +34,7 @@ impl Counter { pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; /// Constructs a new `Counter` of kind `CounterValueReference`. - pub fn counter_value_reference(counter_id: CounterId) -> Self { + pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self { Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } } @@ -59,7 +57,7 @@ impl Counter { /// Must match the layout of `LLVMRustCounterExprKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub enum ExprKind { +pub(crate) enum ExprKind { Subtract = 0, Add = 1, } @@ -69,48 +67,13 @@ pub enum ExprKind { /// Must match the layout of `LLVMRustCounterExpression`. #[derive(Copy, Clone, Debug)] #[repr(C)] -pub struct CounterExpression { - pub kind: ExprKind, - pub lhs: Counter, - pub rhs: Counter, +pub(crate) struct CounterExpression { + pub(crate) kind: ExprKind, + pub(crate) lhs: Counter, + pub(crate) rhs: Counter, } -/// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegionKind`. -#[derive(Copy, Clone, Debug)] -#[repr(C)] -enum RegionKind { - /// A CodeRegion associates some code with a counter - CodeRegion = 0, - - /// An ExpansionRegion represents a file expansion region that associates - /// a source range with the expansion of a virtual source file, such as - /// for a macro instantiation or #include file. - ExpansionRegion = 1, - - /// A SkippedRegion represents a source range with code that was skipped - /// by a preprocessor or similar means. - SkippedRegion = 2, - - /// A GapRegion is like a CodeRegion, but its count is only set as the - /// line execution count when its the only region in the line. - GapRegion = 3, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion = 4, - - /// A DecisionRegion represents a top-level boolean expression and is - /// associated with a variable length bitmap index and condition number. - MCDCDecisionRegion = 5, - - /// A Branch Region can be extended to include IDs to facilitate MC/DC. - MCDCBranchRegion = 6, -} - -mod mcdc { +pub(crate) mod mcdc { use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; /// Must match the layout of `LLVMRustMCDCDecisionParameters`. @@ -121,8 +84,6 @@ mod mcdc { num_conditions: u16, } - // ConditionId in llvm is `unsigned int` at 18 while `int16_t` at - // [19](https://github.com/llvm/llvm-project/pull/81257). type LLVMConditionId = i16; /// Must match the layout of `LLVMRustMCDCBranchParameters`. @@ -133,38 +94,6 @@ mod mcdc { condition_ids: [LLVMConditionId; 2], } - #[repr(C)] - #[derive(Clone, Copy, Debug)] - enum ParameterTag { - None = 0, - Decision = 1, - Branch = 2, - } - /// Same layout with `LLVMRustMCDCParameters` - #[repr(C)] - #[derive(Clone, Copy, Debug)] - pub(crate) struct Parameters { - tag: ParameterTag, - decision_params: DecisionParameters, - branch_params: BranchParameters, - } - - impl Parameters { - pub(crate) fn none() -> Self { - Self { - tag: ParameterTag::None, - decision_params: Default::default(), - branch_params: Default::default(), - } - } - pub(crate) fn decision(decision_params: DecisionParameters) -> Self { - Self { tag: ParameterTag::Decision, decision_params, branch_params: Default::default() } - } - pub(crate) fn branch(branch_params: BranchParameters) -> Self { - Self { tag: ParameterTag::Branch, decision_params: Default::default(), branch_params } - } - } - impl From<ConditionInfo> for BranchParameters { fn from(value: ConditionInfo) -> Self { let to_llvm_cond_id = |cond_id: Option<ConditionId>| { @@ -186,267 +115,68 @@ mod mcdc { } } -/// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the -/// coverage map, in accordance with the -/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). -/// The struct composes fields representing the `Counter` type and value(s) (injected counter -/// ID, or expression type and operands), the source file (an indirect index into a "filenames -/// array", encoded separately), and source location (start and end positions of the represented -/// code region). +/// A span of source code coordinates to be embedded in coverage metadata. /// -/// Corresponds to struct `llvm::coverage::CounterMappingRegion`. -/// -/// Must match the layout of `LLVMRustCounterMappingRegion`. -#[derive(Copy, Clone, Debug)] +/// Must match the layout of `LLVMRustCoverageSpan`. +#[derive(Clone, Debug)] #[repr(C)] -pub struct CounterMappingRegion { - /// The counter type and type-dependent counter data, if any. - counter: Counter, - - /// If the `RegionKind` is a `BranchRegion`, this represents the counter - /// for the false branch of the region. - false_counter: Counter, - - mcdc_params: mcdc::Parameters, - /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the - /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes - /// that, in turn, are used to look up the filename for this region. +pub(crate) struct CoverageSpan { + /// Local index into the function's local-to-global file ID table. + /// The value at that index is itself an index into the coverage filename + /// table in the CGU's `__llvm_covmap` section. file_id: u32, - /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find - /// the mapping regions created as a result of macro expansion, by checking if their file id - /// matches the expanded file id. - expanded_file_id: u32, - - /// 1-based starting line of the mapping region. + /// 1-based starting line of the source code span. start_line: u32, - - /// 1-based starting column of the mapping region. + /// 1-based starting column of the source code span. start_col: u32, - - /// 1-based ending line of the mapping region. + /// 1-based ending line of the source code span. end_line: u32, - - /// 1-based ending column of the mapping region. If the high bit is set, the current - /// mapping region is a gap area. + /// 1-based ending column of the source code span. High bit must be unset. end_col: u32, - - kind: RegionKind, } -impl CounterMappingRegion { - pub(crate) fn from_mapping( - mapping_kind: &MappingKind, - local_file_id: u32, - source_region: &SourceRegion, - ) -> Self { - let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = - source_region; - match *mapping_kind { - MappingKind::Code(term) => Self::code_region( - Counter::from_term(term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::Branch { true_term, false_term } => Self::branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { - Self::mcdc_branch_region( - Counter::from_term(true_term), - Counter::from_term(false_term), - mcdc_params, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ) - } - MappingKind::MCDCDecision(decision_info) => Self::decision_region( - decision_info, - local_file_id, - start_line, - start_col, - end_line, - end_col, - ), - } - } - - pub(crate) fn code_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::CodeRegion, - } +impl CoverageSpan { + pub(crate) fn from_source_region(file_id: u32, code_region: &SourceRegion) -> Self { + let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; + // Internally, LLVM uses the high bit of `end_col` to distinguish between + // code regions and gap regions, so it can't be used by the column number. + assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}"); + Self { file_id, start_line, start_col, end_line, end_col } } +} - pub(crate) fn branch_region( - counter: Counter, - false_counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::BranchRegion, - } - } - - pub(crate) fn mcdc_branch_region( - counter: Counter, - false_counter: Counter, - condition_info: ConditionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter, - mcdc_params: mcdc::Parameters::branch(condition_info.into()), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCBranchRegion, - } - } - - pub(crate) fn decision_region( - decision_info: DecisionInfo, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - let mcdc_params = mcdc::Parameters::decision(decision_info.into()); - - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params, - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::MCDCDecisionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageCodeRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct CodeRegion { + pub(crate) span: CoverageSpan, + pub(crate) counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn expansion_region( - file_id: u32, - expanded_file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::ExpansionRegion, - } - } +/// Must match the layout of `LLVMRustCoverageBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct BranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn skipped_region( - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter: Counter::ZERO, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col, - kind: RegionKind::SkippedRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCBranchRegion { + pub(crate) span: CoverageSpan, + pub(crate) true_counter: Counter, + pub(crate) false_counter: Counter, + pub(crate) mcdc_branch_params: mcdc::BranchParameters, +} - // This function might be used in the future; the LLVM API is still evolving, as is coverage - // support. - #[allow(dead_code)] - pub(crate) fn gap_region( - counter: Counter, - file_id: u32, - start_line: u32, - start_col: u32, - end_line: u32, - end_col: u32, - ) -> Self { - Self { - counter, - false_counter: Counter::ZERO, - mcdc_params: mcdc::Parameters::none(), - file_id, - expanded_file_id: 0, - start_line, - start_col, - end_line, - end_col: (1_u32 << 31) | end_col, - kind: RegionKind::GapRegion, - } - } +/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. +#[derive(Clone, Debug)] +#[repr(C)] +pub(crate) struct MCDCDecisionRegion { + pub(crate) span: CoverageSpan, + pub(crate) mcdc_decision_params: mcdc::DecisionParameters, } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index cee704a9c22..8edd788ee36 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,18 +1,23 @@ -use std::ffi::CStr; +use std::ffi::CString; use itertools::Itertools as _; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods}; +use rustc_abi::Align; +use rustc_codegen_ssa::traits::{ + BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, +}; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_index::IndexVec; +use rustc_middle::mir::coverage::MappingKind; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_span::Symbol; use rustc_span::def_id::DefIdSet; +use rustc_target::spec::HasTargetSpec; use tracing::debug; use crate::common::CodegenCx; -use crate::coverageinfo::ffi::CounterMappingRegion; +use crate::coverageinfo::ffi; use crate::coverageinfo::map_data::{FunctionCoverage, FunctionCoverageCollector}; use crate::{coverageinfo, llvm}; @@ -49,11 +54,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { add_unused_functions(cx); } - let function_coverage_map = match cx.coverage_context() { - Some(ctx) => ctx.take_function_coverage_map(), - None => return, - }; - + let function_coverage_map = cx.coverage_cx().take_function_coverage_map(); if function_coverage_map.is_empty() { // This module has no functions with coverage instrumentation return; @@ -77,11 +78,9 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { // Generate the coverage map header, which contains the filenames used by // this CGU's coverage mappings, and store it in a well-known global. - let cov_data_val = generate_coverage_map(cx, covmap_version, filenames_size, filenames_val); - coverageinfo::save_cov_data_to_mod(cx, cov_data_val); + generate_covmap_record(cx, covmap_version, filenames_size, filenames_val); let mut unused_function_names = Vec::new(); - let covfun_section_name = coverageinfo::covfun_section_name(cx); // Encode coverage mappings and generate function records for (instance, function_coverage) in function_coverage_entries { @@ -110,9 +109,8 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { unused_function_names.push(mangled_function_name); } - save_function_record( + generate_covfun_record( cx, - &covfun_section_name, mangled_function_name, source_hash, filenames_ref, @@ -237,7 +235,10 @@ fn encode_mappings_for_function( let expressions = function_coverage.counter_expressions().collect::<Vec<_>>(); let mut virtual_file_mapping = VirtualFileMapping::default(); - let mut mapping_regions = Vec::with_capacity(counter_regions.len()); + let mut code_regions = vec![]; + let mut branch_regions = vec![]; + let mut mcdc_branch_regions = vec![]; + let mut mcdc_decision_regions = vec![]; // Group mappings into runs with the same filename, preserving the order // yielded by `FunctionCoverage`. @@ -257,11 +258,36 @@ fn encode_mappings_for_function( // form suitable for FFI. for (mapping_kind, region) in counter_regions_for_file { debug!("Adding counter {mapping_kind:?} to map for {region:?}"); - mapping_regions.push(CounterMappingRegion::from_mapping( - &mapping_kind, - local_file_id.as_u32(), - region, - )); + let span = ffi::CoverageSpan::from_source_region(local_file_id.as_u32(), region); + match mapping_kind { + MappingKind::Code(term) => { + code_regions + .push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) }); + } + MappingKind::Branch { true_term, false_term } => { + branch_regions.push(ffi::BranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + }); + } + MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => { + mcdc_branch_regions.push(ffi::MCDCBranchRegion { + span, + true_counter: ffi::Counter::from_term(true_term), + false_counter: ffi::Counter::from_term(false_term), + mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), + }); + } + MappingKind::MCDCDecision(mcdc_decision_params) => { + mcdc_decision_regions.push(ffi::MCDCDecisionRegion { + span, + mcdc_decision_params: ffi::mcdc::DecisionParameters::from( + mcdc_decision_params, + ), + }); + } + } } } @@ -270,21 +296,24 @@ fn encode_mappings_for_function( coverageinfo::write_mapping_to_buffer( virtual_file_mapping.into_vec(), expressions, - mapping_regions, + &code_regions, + &branch_regions, + &mcdc_branch_regions, + &mcdc_decision_regions, buffer, ); }) } -/// Construct coverage map header and the array of function records, and combine them into the -/// coverage map. Save the coverage map data into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn generate_coverage_map<'ll>( +/// Generates the contents of the covmap record for this CGU, which mostly +/// consists of a header and a list of filenames. The record is then stored +/// as a global variable in the `__llvm_covmap` section. +fn generate_covmap_record<'ll>( cx: &CodegenCx<'ll, '_>, version: u32, filenames_size: usize, filenames_val: &'ll llvm::Value, -) -> &'ll llvm::Value { +) { debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); // Create the coverage data header (Note, fields 0 and 2 are now always zero, @@ -299,15 +328,37 @@ fn generate_coverage_map<'ll>( ); // Create the complete LLVM coverage data value to add to the LLVM IR - cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + let covmap_data = + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false); + + let covmap_var_name = CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteMappingVarNameToString(s); + })) + .unwrap(); + debug!("covmap var name: {:?}", covmap_var_name); + + let covmap_section_name = CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); + })) + .expect("covmap section name should not contain NUL"); + debug!("covmap section name: {:?}", covmap_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(covmap_data), &covmap_var_name); + llvm::set_initializer(llglobal, covmap_data); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, &covmap_section_name); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + cx.add_used_global(llglobal); } -/// Construct a function record and combine it with the function's coverage mapping data. -/// Save the function record into the LLVM IR as a static global using a -/// specific, well-known section and name. -fn save_function_record( +/// Generates the contents of the covfun record for this function, which +/// contains the function's coverage mapping data. The record is then stored +/// as a global variable in the `__llvm_covfun` section. +fn generate_covfun_record( cx: &CodegenCx<'_, '_>, - covfun_section_name: &CStr, mangled_function_name: &str, source_hash: u64, filenames_ref: u64, @@ -334,13 +385,28 @@ fn save_function_record( /*packed=*/ true, ); - coverageinfo::save_func_record_to_mod( - cx, - covfun_section_name, - func_name_hash, - func_record_val, - is_used, - ); + // Choose a variable name to hold this function's covfun data. + // Functions that are used have a suffix ("u") to distinguish them from + // unused copies of the same function (from different CGUs), so that if a + // linker sees both it won't discard the used copy's data. + let func_record_var_name = + CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) + .unwrap(); + debug!("function record var name: {:?}", func_record_var_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, cx.covfun_section_name()); + // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. + // <https://llvm.org/docs/CoverageMappingFormat.html> + llvm::set_alignment(llglobal, Align::EIGHT); + if cx.target_spec().supports_comdat() { + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + } + cx.add_used_global(llglobal); } /// Each CGU will normally only emit coverage metadata for the functions that it actually generates. @@ -472,9 +538,5 @@ fn add_unused_function_coverage<'tcx>( // zero, because none of its counters/expressions are marked as seen. let function_coverage = FunctionCoverageCollector::unused(instance, function_coverage_info); - if let Some(coverage_context) = cx.coverage_context() { - coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); - } else { - bug!("Could not get the `coverage_context`"); - } + cx.coverage_cx().function_coverage_map.borrow_mut().insert(instance, function_coverage); } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 484a4d00c13..a298ed86276 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,24 +1,20 @@ -use std::cell::RefCell; +use std::cell::{OnceCell, RefCell}; use std::ffi::{CStr, CString}; use libc::c_uint; use rustc_codegen_ssa::traits::{ - BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, - MiscCodegenMethods, StaticCodegenMethods, + BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; -use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use rustc_middle::ty::layout::HasTyCtxt; -use rustc_target::abi::{Align, Size}; -use rustc_target::spec::HasTargetSpec; +use rustc_target::abi::Size; use tracing::{debug, instrument}; use crate::builder::Builder; -use crate::common::CodegenCx; -use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion}; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::coverageinfo::map_data::FunctionCoverageCollector; use crate::llvm; @@ -33,6 +29,8 @@ pub(crate) struct CrateCoverageContext<'ll, 'tcx> { RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>, pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>, + + covfun_section_name: OnceCell<CString>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -41,6 +39,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), mcdc_condition_bitmap_map: Default::default(), + covfun_section_name: Default::default(), } } @@ -67,27 +66,38 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { } } -// These methods used to be part of trait `CoverageInfoMethods`, which no longer -// exists after most coverage code was moved out of SSA. impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn coverageinfo_finalize(&self) { mapgen::finalize(self) } + /// Returns the section name to use when embedding per-function coverage information + /// in the object file, according to the target's object file format. LLVM's coverage + /// tools use information from this section when producing coverage reports. + /// + /// Typical values are: + /// - `__llvm_covfun` on Linux + /// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) + /// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) + fn covfun_section_name(&self) -> &CStr { + self.coverage_cx().covfun_section_name.get_or_init(|| { + CString::new(llvm::build_byte_buffer(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(self.llmod, s); + })) + .expect("covfun section name should not contain NUL") + }) + } + /// For LLVM codegen, returns a function-specific `Value` for a global /// string, to hold the function name passed to LLVM intrinsic /// `instrprof.increment()`. The `Value` is only created once per instance. /// Multiple invocations with the same instance return the same `Value`. fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { - if let Some(coverage_context) = self.coverage_context() { - debug!("getting pgo_func_name_var for instance={:?}", instance); - let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); - pgo_func_name_var_map - .entry(instance) - .or_insert_with(|| create_pgo_func_name_var(self, instance)) - } else { - bug!("Could not get the `coverage_context`"); - } + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = self.coverage_cx().pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map + .entry(instance) + .or_insert_with(|| create_pgo_func_name_var(self, instance)) } } @@ -121,11 +131,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { cond_bitmaps.push(cond_bitmap); } - self.coverage_context() - .expect("always present when coverage is enabled") - .mcdc_condition_bitmap_map - .borrow_mut() - .insert(instance, cond_bitmaps); + self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps); } #[instrument(level = "debug", skip(self))] @@ -146,8 +152,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; - let Some(coverage_context) = bx.coverage_context() else { return }; - let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + let mut coverage_map = bx.coverage_cx().function_coverage_map.borrow_mut(); let func_coverage = coverage_map .entry(instance) .or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info)); @@ -189,7 +194,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } CoverageKind::CondBitmapUpdate { index, decision_depth } => { drop(coverage_map); - let cond_bitmap = coverage_context + let cond_bitmap = bx + .coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for updating"); let cond_index = bx.const_i32(index as i32); @@ -197,7 +203,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { drop(coverage_map); - let cond_bitmap = coverage_context + let cond_bitmap = bx.coverage_cx() .try_get_mcdc_condition_bitmap(&instance, decision_depth) .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); assert!( @@ -209,6 +215,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); let bitmap_index = bx.const_u32(bitmap_idx); bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); + bx.mcdc_condbitmap_reset(cond_bitmap); } } } @@ -229,7 +236,7 @@ fn create_pgo_func_name_var<'ll, 'tcx>( unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar( llfn, - mangled_fn_name.as_ptr().cast(), + mangled_fn_name.as_c_char_ptr(), mangled_fn_name.len(), ) } @@ -241,7 +248,7 @@ pub(crate) fn write_filenames_section_to_buffer<'a>( ) { let (pointers, lengths) = filenames .into_iter() - .map(|s: &str| (s.as_ptr().cast(), s.len())) + .map(|s: &str| (s.as_c_char_ptr(), s.len())) .unzip::<_, _, Vec<_>, Vec<_>>(); unsafe { @@ -257,8 +264,11 @@ pub(crate) fn write_filenames_section_to_buffer<'a>( pub(crate) fn write_mapping_to_buffer( virtual_file_mapping: Vec<u32>, - expressions: Vec<CounterExpression>, - mapping_regions: Vec<CounterMappingRegion>, + expressions: Vec<ffi::CounterExpression>, + code_regions: &[ffi::CodeRegion], + branch_regions: &[ffi::BranchRegion], + mcdc_branch_regions: &[ffi::MCDCBranchRegion], + mcdc_decision_regions: &[ffi::MCDCDecisionRegion], buffer: &RustString, ) { unsafe { @@ -267,96 +277,23 @@ pub(crate) fn write_mapping_to_buffer( virtual_file_mapping.len() as c_uint, expressions.as_ptr(), expressions.len() as c_uint, - mapping_regions.as_ptr(), - mapping_regions.len() as c_uint, + code_regions.as_ptr(), + code_regions.len() as c_uint, + branch_regions.as_ptr(), + branch_regions.len() as c_uint, + mcdc_branch_regions.as_ptr(), + mcdc_branch_regions.len() as c_uint, + mcdc_decision_regions.as_ptr(), + mcdc_decision_regions.len() as c_uint, buffer, ); } } pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } + unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_c_char_ptr(), bytes.len()) } } pub(crate) fn mapping_version() -> u32 { unsafe { llvm::LLVMRustCoverageMappingVersion() } } - -pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - cov_data_val: &'ll llvm::Value, -) { - let covmap_var_name = CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteMappingVarNameToString(s); - })) - .unwrap(); - debug!("covmap var name: {:?}", covmap_var_name); - - let covmap_section_name = CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); - })) - .expect("covmap section name should not contain NUL"); - debug!("covmap section name: {:?}", covmap_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); - llvm::set_initializer(llglobal, cov_data_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::set_section(llglobal, &covmap_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - cx.add_used_global(llglobal); -} - -pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, - covfun_section_name: &CStr, - func_name_hash: u64, - func_record_val: &'ll llvm::Value, - is_used: bool, -) { - // Assign a name to the function record. This is used to merge duplicates. - // - // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that - // are included-but-not-used. If (or when) Rust generates functions that are - // included-but-not-used, note that a dummy description for a function included-but-not-used - // in a Crate can be replaced by full description provided by a different Crate. The two kinds - // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by - // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. - let func_record_var_name = - CString::new(format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" })) - .unwrap(); - debug!("function record var name: {:?}", func_record_var_name); - debug!("function record section name: {:?}", covfun_section_name); - - let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); - llvm::set_initializer(llglobal, func_record_val); - llvm::set_global_constant(llglobal, true); - llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); - llvm::set_visibility(llglobal, llvm::Visibility::Hidden); - llvm::set_section(llglobal, covfun_section_name); - // LLVM's coverage mapping format specifies 8-byte alignment for items in this section. - llvm::set_alignment(llglobal, Align::EIGHT); - if cx.target_spec().supports_comdat() { - llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); - } - cx.add_used_global(llglobal); -} - -/// Returns the section name string to pass through to the linker when embedding -/// per-function coverage information in the object file, according to the target -/// platform's object file format. -/// -/// LLVM's coverage tools read coverage mapping details from this section when -/// producing coverage reports. -/// -/// Typical values are: -/// - `__llvm_covfun` on Linux -/// - `__LLVM_COV,__llvm_covfun` on macOS (includes `__LLVM_COV,` segment prefix) -/// - `.lcovfun$M` on Windows (includes `$M` sorting suffix) -pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { - llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); - })) - .expect("covfun section name should not contain NUL") -} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index f93d3e40b20..7947c9c8c8e 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -76,7 +76,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); llvm::LLVMSetGlobalConstant(section_var, llvm::True); llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); - llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than // the string it contains. Otherwise we get a warning from GDB. llvm::LLVMSetAlignment(section_var, 1); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 15d441a986d..9064cfaeb29 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -32,7 +32,7 @@ use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_na use super::utils::{ DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, }; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::build_type_with_children; use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; use crate::llvm::debuginfo::{ @@ -190,7 +190,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( data_layout.pointer_size.bits(), data_layout.pointer_align.abi.bits() as u32, 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.as_c_char_ptr(), ptr_type_debuginfo_name.len(), ) }; @@ -348,7 +348,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( size, align, 0, // Ignore DWARF address space. - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), ) }; @@ -518,7 +518,7 @@ fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll D let name = "<recur_type>"; llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.tcx.data_layout.pointer_size.bits(), DW_ATE_unsigned, @@ -640,14 +640,14 @@ pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFi unsafe { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), hash_kind, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), - source.map_or(ptr::null(), |x| x.as_ptr().cast()), + source.map_or(ptr::null(), |x| x.as_c_char_ptr()), source.map_or(0, |x| x.len()), ) } @@ -662,12 +662,12 @@ fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { llvm::LLVMRustDIBuilderCreateFile( DIB(cx), - file_name.as_ptr().cast(), + file_name.as_c_char_ptr(), file_name.len(), - directory.as_ptr().cast(), + directory.as_c_char_ptr(), directory.len(), llvm::ChecksumKind::None, - hash_value.as_ptr().cast(), + hash_value.as_c_char_ptr(), hash_value.len(), ptr::null(), 0, @@ -788,7 +788,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( let ty_di_node = unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), cx.size_of(t).bits(), encoding, @@ -810,7 +810,7 @@ fn build_basic_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_ptr().cast(), + typedef_name.as_c_char_ptr(), typedef_name.len(), unknown_file_metadata(cx), 0, @@ -861,7 +861,7 @@ fn build_param_type_di_node<'ll, 'tcx>( di_node: unsafe { llvm::LLVMRustDIBuilderCreateBasicType( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), Size::ZERO.bits(), DW_ATE_unsigned, @@ -948,9 +948,9 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( unsafe { let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( debug_context.builder, - name_in_debuginfo.as_ptr().cast(), + name_in_debuginfo.as_c_char_ptr(), name_in_debuginfo.len(), - work_dir.as_ptr().cast(), + work_dir.as_c_char_ptr(), work_dir.len(), llvm::ChecksumKind::None, ptr::null(), @@ -963,7 +963,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( debug_context.builder, DW_LANG_RUST, compile_unit_file, - producer.as_ptr().cast(), + producer.as_c_char_ptr(), producer.len(), tcx.sess.opts.optimize != config::OptLevel::No, c"".as_ptr(), @@ -971,7 +971,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead // put the path supplied to `MCSplitDwarfFile` into the debug info of the final // output(s). - split_name.as_ptr().cast(), + split_name.as_c_char_ptr(), split_name.len(), kind, 0, @@ -1022,7 +1022,7 @@ fn build_field_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), owner, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -1306,7 +1306,7 @@ fn build_generic_type_param_di_nodes<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_di_node, ) @@ -1382,9 +1382,9 @@ pub(crate) fn build_global_var_di_node<'ll>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), Some(var_scope), - var_name.as_ptr().cast(), + var_name.as_c_char_ptr(), var_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, line_number, @@ -1602,9 +1602,9 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticVariable( DIB(cx), NO_SCOPE_METADATA, - vtable_name.as_ptr().cast(), + vtable_name.as_c_char_ptr(), vtable_name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 966788cf32f..5385d3a9212 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; use rustc_target::abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; use crate::debuginfo::metadata::{ @@ -359,7 +359,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), enum_type_di_node, - TAG_FIELD_NAME.as_ptr().cast(), + TAG_FIELD_NAME.as_c_char_ptr(), TAG_FIELD_NAME.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -537,7 +537,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStaticMemberType( DIB(cx), wrapper_struct_type_di_node, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -785,7 +785,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), enum_type_di_node, - field_name.as_ptr().cast(), + field_name.as_c_char_ptr(), field_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index fe1634146ff..4c848027b55 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -13,7 +13,7 @@ use rustc_target::abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub}; use crate::debuginfo::metadata::{ UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, type_di_node, @@ -106,7 +106,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( let value = [value as u64, (value >> 64) as u64]; Some(llvm::LLVMRustDIBuilderCreateEnumerator( DIB(cx), - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), value.as_ptr(), size.bits() as libc::c_uint, @@ -119,7 +119,7 @@ fn build_enumeration_type_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateEnumerationType( DIB(cx), containing_scope, - type_name.as_ptr().cast(), + type_name.as_c_char_ptr(), type_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 5e7dbdd921a..b7400c5aeb2 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -10,7 +10,7 @@ use rustc_middle::ty::{self}; use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; use smallvec::smallvec; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::metadata::type_map::{self, Stub, StubInfo, UniqueTypeId}; use crate::debuginfo::metadata::{ DINodeCreationResult, NO_GENERICS, SmallVec, UNKNOWN_LINE_NUMBER, file_metadata, @@ -244,7 +244,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantPart( DIB(cx), enum_type_di_node, - variant_part_name.as_ptr().cast(), + variant_part_name.as_c_char_ptr(), variant_part_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -253,7 +253,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), - variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.as_c_char_ptr(), variant_part_unique_type_id_str.len(), ) }, @@ -327,7 +327,7 @@ fn build_discr_member_di_node<'ll, 'tcx>( Some(llvm::LLVMRustDIBuilderCreateMemberType( DIB(cx), containing_scope, - tag_name.as_ptr().cast(), + tag_name.as_c_char_ptr(), tag_name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -399,7 +399,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateVariantMemberType( DIB(cx), variant_part_di_node, - variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.as_c_char_ptr(), variant_member_info.variant_name.len(), file_di_node, line_number, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index 714e3c0b145..d050dc9b406 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -9,7 +9,7 @@ use rustc_middle::ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}; use rustc_target::abi::{Align, Size, VariantIdx}; use super::{SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -191,7 +191,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -202,7 +202,7 @@ pub(super) fn stub<'ll, 'tcx>( empty_array, 0, vtable_holder, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) } @@ -211,7 +211,7 @@ pub(super) fn stub<'ll, 'tcx>( llvm::LLVMRustDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, @@ -220,7 +220,7 @@ pub(super) fn stub<'ll, 'tcx>( flags, Some(empty_array), 0, - unique_type_id_str.as_ptr().cast(), + unique_type_id_str.as_c_char_ptr(), unique_type_id_str.len(), ) }, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 1a8153a54e8..72e723aa849 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -31,7 +31,7 @@ use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::abi::FnAbi; use crate::builder::Builder; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::{ DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, @@ -91,45 +91,39 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { } pub(crate) fn finalize(&self, sess: &Session) { - unsafe { - llvm::LLVMRustDIBuilderFinalize(self.builder); - - if !sess.target.is_like_msvc { - // Debuginfo generation in LLVM by default uses a higher - // version of dwarf than macOS currently understands. We can - // instruct LLVM to emit an older version of dwarf, however, - // for macOS to understand. For more info see #11352 - // This can be overridden using --llvm-opts -dwarf-version,N. - // Android has the same issue (#22398) - let dwarf_version = sess - .opts - .unstable_opts - .dwarf_version - .unwrap_or(sess.target.default_dwarf_version); - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Dwarf Version".as_ptr(), - dwarf_version, - ); - } else { - // Indicate that we want CodeView debug information on MSVC - llvm::LLVMRustAddModuleFlagU32( - self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"CodeView".as_ptr(), - 1, - ) - } - - // Prevent bitcode readers from deleting the debug info. - llvm::LLVMRustAddModuleFlagU32( + unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) }; + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = + sess.opts.unstable_opts.dwarf_version.unwrap_or(sess.target.default_dwarf_version); + llvm::add_module_flag_u32( self.llmod, - llvm::LLVMModFlagBehavior::Warning, - c"Debug Info Version".as_ptr(), - llvm::LLVMRustDebugMetadataVersion(), + llvm::ModuleFlagMergeBehavior::Warning, + "Dwarf Version", + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "CodeView", + 1, ); } + + // Prevent bitcode readers from deleting the debug info. + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Warning, + "Debug Info Version", + unsafe { llvm::LLVMRustDebugMetadataVersion() }, + ); } } @@ -350,7 +344,6 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { type_names::push_generic_params( tcx, tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), args), - enclosing_fn_def_id, &mut name, ); @@ -365,7 +358,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let mut flags = DIFlags::FlagPrototyped; - if fn_abi.ret.layout.abi.is_uninhabited() { + if fn_abi.ret.layout.is_uninhabited() { flags |= DIFlags::FlagNoReturn; } @@ -390,9 +383,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateMethod( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -407,9 +400,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { llvm::LLVMRustDIBuilderCreateFunction( DIB(self), containing_scope, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), - linkage_name.as_ptr().cast(), + linkage_name.as_c_char_ptr(), linkage_name.len(), file_metadata, loc.line, @@ -495,7 +488,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( DIB(cx), None, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), actual_type_metadata, )) @@ -636,7 +629,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { DIB(self), dwarf_tag, scope_metadata, - name.as_ptr().cast(), + name.as_c_char_ptr(), name.len(), file_metadata, loc.line, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index 3578755aae0..33d9bc23890 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -5,7 +5,7 @@ use rustc_hir::def_id::DefId; use rustc_middle::ty::{self, Instance}; use super::utils::{DIB, debug_context}; -use crate::common::CodegenCx; +use crate::common::{AsCCharPtr, CodegenCx}; use crate::llvm; use crate::llvm::debuginfo::DIScope; @@ -36,7 +36,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l llvm::LLVMRustDIBuilderCreateNameSpace( DIB(cx), parent_scope, - namespace_name_string.as_ptr().cast(), + namespace_name_string.as_c_char_ptr(), namespace_name_string.len(), false, // ExportSymbols (only relevant for C++ anonymous namespaces) ) diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 33258cb46fa..d338c848754 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -20,6 +20,7 @@ use smallvec::SmallVec; use tracing::debug; use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::common::AsCCharPtr; use crate::context::CodegenCx; use crate::llvm::AttributePlace::Function; use crate::llvm::Visibility; @@ -41,7 +42,7 @@ fn declare_raw_fn<'ll>( ) -> &'ll Value { debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); let llfn = unsafe { - llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) + llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_c_char_ptr(), name.len(), ty) }; llvm::SetFunctionCallConv(llfn, callconv); @@ -68,7 +69,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// return its Value instead. pub(crate) fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { debug!("declare_global(name={:?})", name); - unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } + unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_c_char_ptr(), name.len(), ty) } } /// Declare a C ABI function. @@ -209,7 +210,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); - unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_c_char_ptr(), name.len()) } } /// Gets defined or externally defined (AvailableExternally linkage) value by diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c9a17c9852d..d04b5257619 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -785,13 +785,12 @@ fn codegen_msvc_try<'ll>( let type_info = bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false); let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); - unsafe { - llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); - if bx.cx.tcx.sess.target.supports_comdat() { - llvm::SetUniqueComdat(bx.llmod, tydesc); - } - llvm::LLVMSetInitializer(tydesc, type_info); + + llvm::set_linkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + if bx.cx.tcx.sess.target.supports_comdat() { + llvm::SetUniqueComdat(bx.llmod, tydesc); } + unsafe { llvm::LLVMSetInitializer(tydesc, type_info) }; // The flag value of 8 indicates that we are catching the exception by // reference instead of by value. We can't use catch by value because @@ -1064,7 +1063,7 @@ fn gen_fn<'ll, 'tcx>( cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. - unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; + llvm::set_linkage(llfn, llvm::Linkage::InternalLinkage); let llbb = Builder::append_block(cx, llfn, "entry-block"); let bx = Builder::build(cx, llbb); codegen(bx); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index d0034de06c7..888d41d4726 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1,9 +1,11 @@ #![allow(non_camel_case_types)] #![allow(non_upper_case_globals)] +use std::fmt::Debug; use std::marker::PhantomData; use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; +use rustc_macros::TryFromU32; use rustc_target::spec::SymbolVisibility; use super::RustString; @@ -19,6 +21,30 @@ pub type Bool = c_uint; pub const True: Bool = 1 as Bool; pub const False: Bool = 0 as Bool; +/// Wrapper for a raw enum value returned from LLVM's C APIs. +/// +/// For C enums returned by LLVM, it's risky to use a Rust enum as the return +/// type, because it would be UB if a later version of LLVM adds a new enum +/// value and returns it. Instead, return this raw wrapper, then convert to the +/// Rust-side enum explicitly. +#[repr(transparent)] +pub struct RawEnum<T> { + value: u32, + /// We don't own or consume a `T`, but we can produce one. + _rust_side_type: PhantomData<fn() -> T>, +} + +impl<T: TryFrom<u32>> RawEnum<T> { + #[track_caller] + pub(crate) fn to_rust(self) -> T + where + T::Error: Debug, + { + // If this fails, the Rust-side enum is out of sync with LLVM's enum. + T::try_from(self.value).expect("enum value returned by LLVM should be known") + } +} + #[derive(Copy, Clone, PartialEq)] #[repr(C)] #[allow(dead_code)] // Variants constructed by C++. @@ -59,7 +85,7 @@ pub enum LLVMMachineType { ARM = 0x01c0, } -/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// Must match the layout of `LLVMRustModuleFlagMergeBehavior`. /// /// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are /// resolved according to the merge behaviors specified here. Flags differing only in merge @@ -67,9 +93,13 @@ pub enum LLVMMachineType { /// /// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, /// 'Error' and 'Warning' cannot be mixed for a given flag. +/// +/// There is a stable LLVM-C version of this enum (`LLVMModuleFlagBehavior`), +/// but as of LLVM 19 it does not support all of the enum values in the unstable +/// C++ API. #[derive(Copy, Clone, PartialEq)] #[repr(C)] -pub enum LLVMModFlagBehavior { +pub enum ModuleFlagMergeBehavior { Error = 1, Warning = 2, Require = 3, @@ -108,26 +138,36 @@ pub enum CallConv { AvrInterrupt = 85, } -/// LLVMRustLinkage -#[derive(Copy, Clone, PartialEq)] +/// Must match the layout of `LLVMLinkage`. +#[derive(Copy, Clone, PartialEq, TryFromU32)] #[repr(C)] pub enum Linkage { ExternalLinkage = 0, AvailableExternallyLinkage = 1, LinkOnceAnyLinkage = 2, LinkOnceODRLinkage = 3, - WeakAnyLinkage = 4, - WeakODRLinkage = 5, - AppendingLinkage = 6, - InternalLinkage = 7, - PrivateLinkage = 8, - ExternalWeakLinkage = 9, - CommonLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + LinkOnceODRAutoHideLinkage = 4, + WeakAnyLinkage = 5, + WeakODRLinkage = 6, + AppendingLinkage = 7, + InternalLinkage = 8, + PrivateLinkage = 9, + #[deprecated = "marked obsolete by LLVM"] + DLLImportLinkage = 10, + #[deprecated = "marked obsolete by LLVM"] + DLLExportLinkage = 11, + ExternalWeakLinkage = 12, + #[deprecated = "marked obsolete by LLVM"] + GhostLinkage = 13, + CommonLinkage = 14, + LinkerPrivateLinkage = 15, + LinkerPrivateWeakLinkage = 16, } -// LLVMRustVisibility +/// Must match the layout of `LLVMVisibility`. #[repr(C)] -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, TryFromU32)] pub enum Visibility { Default = 0, Hidden = 1, @@ -945,7 +985,11 @@ unsafe extern "C" { // Operations on global variables, functions, and aliases (globals) pub fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub fn LLVMGetLinkage(Global: &Value) -> RawEnum<Linkage>; + pub fn LLVMSetLinkage(Global: &Value, RustLinkage: Linkage); pub fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub fn LLVMGetVisibility(Global: &Value) -> RawEnum<Visibility>; + pub fn LLVMSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMGetAlignment(Global: &Value) -> c_uint; pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); @@ -1521,10 +1565,6 @@ unsafe extern "C" { ) -> bool; // Operations on global variables, functions, and aliases (globals) - pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; - pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); - pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; - pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); // Operations on global variables @@ -1615,10 +1655,6 @@ unsafe extern "C" { pub fn LLVMRustSetAllowReassoc(Instr: &Value); // Miscellaneous instructions - pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; - pub fn LLVMRustBuildCall<'a>( B: &Builder<'a>, Ty: &'a Type, @@ -1744,7 +1780,7 @@ unsafe extern "C" { ) -> bool; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + pub(crate) fn LLVMRustCoverageWriteFilenamesSectionToBuffer( Filenames: *const *const c_char, FilenamesLen: size_t, Lengths: *const size_t, @@ -1753,33 +1789,39 @@ unsafe extern "C" { ); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingToBuffer( + pub(crate) fn LLVMRustCoverageWriteMappingToBuffer( VirtualFileMappingIDs: *const c_uint, NumVirtualFileMappingIDs: c_uint, Expressions: *const crate::coverageinfo::ffi::CounterExpression, NumExpressions: c_uint, - MappingRegions: *const crate::coverageinfo::ffi::CounterMappingRegion, - NumMappingRegions: c_uint, + CodeRegions: *const crate::coverageinfo::ffi::CodeRegion, + NumCodeRegions: c_uint, + BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, + NumBranchRegions: c_uint, + MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, + NumMCDCBranchRegions: c_uint, + MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, + NumMCDCDecisionRegions: c_uint, BufferOut: &RustString, ); - pub fn LLVMRustCoverageCreatePGOFuncNameVar( + pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, FuncName: *const c_char, FuncNameLen: size_t, ) -> &Value; - pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); #[allow(improper_ctypes)] - pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + pub(crate) fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); - pub fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; pub fn LLVMRustDebugMetadataVersion() -> u32; pub fn LLVMRustVersionMajor() -> u32; pub fn LLVMRustVersionMinor() -> u32; @@ -1791,21 +1833,21 @@ unsafe extern "C" { /// "compatible" means depends on the merge behaviors involved. pub fn LLVMRustAddModuleFlagU32( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: u32, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: u32, ); pub fn LLVMRustAddModuleFlagString( M: &Module, - merge_behavior: LLVMModFlagBehavior, - name: *const c_char, - value: *const c_char, - value_len: size_t, + MergeBehavior: ModuleFlagMergeBehavior, + Name: *const c_char, + NameLen: size_t, + Value: *const c_char, + ValueLen: size_t, ); - pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; - pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); @@ -2347,9 +2389,9 @@ unsafe extern "C" { pub fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; pub fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, - NumModules: c_uint, + NumModules: size_t, PreservedSymbols: *const *const c_char, - PreservedSymbolsLen: c_uint, + PreservedSymbolsLen: size_t, ) -> Option<&'static mut ThinLTOData>; pub fn LLVMRustPrepareThinLTORename( Data: &ThinLTOData, diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index e837022044e..acd425bbb8e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -17,13 +17,13 @@ pub use self::IntPredicate::*; pub use self::Linkage::*; pub use self::MetadataType::*; pub use self::RealPredicate::*; +pub use self::ffi::*; +use crate::common::AsCCharPtr; pub mod archive_ro; pub mod diagnostic; mod ffi; -pub use self::ffi::*; - impl LLVMRustResult { pub fn into_result(self) -> Result<(), ()> { match self { @@ -53,9 +53,9 @@ pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) - unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), - value.as_ptr().cast(), + value.as_c_char_ptr(), value.len().try_into().unwrap(), ) } @@ -65,7 +65,7 @@ pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { unsafe { LLVMCreateStringAttribute( llcx, - attr.as_ptr().cast(), + attr.as_c_char_ptr(), attr.len().try_into().unwrap(), std::ptr::null(), 0, @@ -232,15 +232,23 @@ pub fn set_global_constant(llglobal: &Value, is_constant: bool) { } } +pub fn get_linkage(llglobal: &Value) -> Linkage { + unsafe { LLVMGetLinkage(llglobal) }.to_rust() +} + pub fn set_linkage(llglobal: &Value, linkage: Linkage) { unsafe { - LLVMRustSetLinkage(llglobal, linkage); + LLVMSetLinkage(llglobal, linkage); } } +pub fn get_visibility(llglobal: &Value) -> Visibility { + unsafe { LLVMGetVisibility(llglobal) }.to_rust() +} + pub fn set_visibility(llglobal: &Value, visibility: Visibility) { unsafe { - LLVMRustSetVisibility(llglobal, visibility); + LLVMSetVisibility(llglobal, visibility); } } @@ -286,7 +294,7 @@ pub fn get_value_name(value: &Value) -> &[u8] { /// Safe wrapper for `LLVMSetValueName2` from a byte slice pub fn set_value_name(value: &Value, name: &[u8]) { unsafe { - let data = name.as_ptr().cast(); + let data = name.as_c_char_ptr(); LLVMSetValueName2(value, data, name.len()); } } @@ -344,3 +352,32 @@ impl Drop for OperandBundleDef<'_> { } } } + +pub(crate) fn add_module_flag_u32( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: u32, +) { + unsafe { + LLVMRustAddModuleFlagU32(module, merge_behavior, key.as_c_char_ptr(), key.len(), value); + } +} + +pub(crate) fn add_module_flag_str( + module: &Module, + merge_behavior: ModuleFlagMergeBehavior, + key: &str, + value: &str, +) { + unsafe { + LLVMRustAddModuleFlagString( + module, + merge_behavior, + key.as_c_char_ptr(), + key.len(), + value.as_c_char_ptr(), + value.len(), + ); + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 57936215ff1..9adb1299b3d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -248,6 +248,7 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea ("aarch64", "pmuv3") => Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), + ("aarch64", "pauth-lr") if get_version().0 < 19 => None, // Before LLVM 20 those two features were packaged together as b16b16 ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), @@ -697,12 +698,9 @@ fn backend_feature_name<'a>(sess: &Session, s: &'a str) -> Option<&'a str> { let feature = s .strip_prefix(&['+', '-'][..]) .unwrap_or_else(|| sess.dcx().emit_fatal(InvalidTargetFeaturePrefix { feature: s })); - if s.is_empty() { - return None; - } // Rustc-specific feature requests like `+crt-static` or `-crt-static` // are not passed down to LLVM. - if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + if s.is_empty() || RUSTC_SPECIFIC_FEATURES.contains(&feature) { return None; } Some(feature) diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index bf6ef219873..ea8857b4739 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -39,9 +39,9 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { .emit_fatal(SymbolAlreadyDefined { span: self.tcx.def_span(def_id), symbol_name }) }); + llvm::set_linkage(g, base::linkage_to_llvm(linkage)); + llvm::set_visibility(g, base::visibility_to_llvm(visibility)); unsafe { - llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); - llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); if self.should_assume_dso_local(g, false) { llvm::LLVMRustSetDSOLocal(g, true); } @@ -61,7 +61,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); - unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; + llvm::set_linkage(lldecl, base::linkage_to_llvm(linkage)); let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); if (linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR) @@ -78,21 +78,15 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { && linkage != Linkage::Private && self.tcx.is_compiler_builtins(LOCAL_CRATE) { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); - } + llvm::set_visibility(lldecl, llvm::Visibility::Hidden); } else { - unsafe { - llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); - } + llvm::set_visibility(lldecl, base::visibility_to_llvm(visibility)); } debug!("predefine_fn: instance = {:?}", instance); - unsafe { - if self.should_assume_dso_local(lldecl, false) { - llvm::LLVMRustSetDSOLocal(lldecl, true); - } + if self.should_assume_dso_local(lldecl, false) { + unsafe { llvm::LLVMRustSetDSOLocal(lldecl, true) }; } self.instances.borrow_mut().insert(instance, lldecl); @@ -102,13 +96,13 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { impl CodegenCx<'_, '_> { /// Whether a definition or declaration can be assumed to be local to a group of /// libraries that form a single DSO or executable. - pub(crate) unsafe fn should_assume_dso_local( + pub(crate) fn should_assume_dso_local( &self, llval: &llvm::Value, is_declaration: bool, ) -> bool { - let linkage = unsafe { llvm::LLVMRustGetLinkage(llval) }; - let visibility = unsafe { llvm::LLVMRustGetVisibility(llval) }; + let linkage = llvm::get_linkage(llval); + let visibility = llvm::get_visibility(llval); if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { return true; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 1af666f818b..6be4c3f034f 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -191,7 +191,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. /// If the type is an unsized struct, the regular layout is generated, - /// with the inner-most trailing unsized field using the "minimal unit" + /// with the innermost trailing unsized field using the "minimal unit" /// of that field's type - this is useful for taking the address of /// that field and ensuring the struct has the right alignment. fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { |