diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/write.rs | 59 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/builder.rs | 11 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs | 34 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/declare.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 10 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 38 |
9 files changed, 132 insertions, 64 deletions
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 4bb1fed2d51..ddba43cd1f1 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -3,7 +3,7 @@ use crate::back::write::{ }; use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{self, build_string, False, True}; -use crate::{LlvmCodegenBackend, ModuleLlvm}; +use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{ @@ -596,7 +596,10 @@ pub(crate) fn run_pass_manager( // tools/lto/LTOCodeGenerator.cpp debug!("running the pass manager"); unsafe { - if write::should_use_new_llvm_pass_manager(cgcx, config) { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); write::optimize_with_new_llvm_pass_manager( diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index fb194a98a0d..384596dfff5 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -23,7 +23,7 @@ use rustc_errors::{FatalError, Handler, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; -use rustc_session::config::{self, Lto, OutputType, Passes, SwitchWithOptPath}; +use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; use rustc_session::Session; use rustc_span::symbol::sym; use rustc_span::InnerSpan; @@ -106,7 +106,11 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm: pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { - tcx.output_filenames(()).split_dwarf_path(tcx.sess.split_debuginfo(), Some(mod_name)) + tcx.output_filenames(()).split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.debugging_opts.split_dwarf_kind, + Some(mod_name), + ) } else { None }; @@ -416,21 +420,6 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> { .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) } -pub(crate) fn should_use_new_llvm_pass_manager( - cgcx: &CodegenContext<LlvmCodegenBackend>, - config: &ModuleConfig, -) -> bool { - // The new pass manager is enabled by default for LLVM >= 13. - // This matches Clang, which also enables it since Clang 13. - - // FIXME: There are some perf issues with the new pass manager - // when targeting s390x, so it is temporarily disabled for that - // arch, see https://github.com/rust-lang/rust/issues/89609 - config - .new_llvm_pass_manager - .unwrap_or_else(|| cgcx.target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0)) -} - pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, @@ -473,6 +462,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( let extra_passes = config.passes.join(","); + let llvm_plugins = config.llvm_plugins.join(","); + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. // We would have to add upstream support for this first, before we can support // config.inline_threshold and our more aggressive default thresholds. @@ -502,6 +493,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( selfprofile_after_pass_callback, extra_passes.as_ptr().cast(), extra_passes.len(), + llvm_plugins.as_ptr().cast(), + llvm_plugins.len(), ); result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) } @@ -530,7 +523,10 @@ pub(crate) unsafe fn optimize( } if let Some(opt_level) = config.opt_level { - if should_use_new_llvm_pass_manager(cgcx, config) { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { let opt_stage = match cgcx.lto { Lto::Fat => llvm::OptStage::PreLinkFatLTO, Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, @@ -900,17 +896,18 @@ pub(crate) unsafe fn codegen( .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); - let dwo_out = match cgcx.split_debuginfo { - // Don't change how DWARF is emitted in single mode (or when disabled). - SplitDebuginfo::Off | SplitDebuginfo::Packed => None, - // Emit (a subset of the) DWARF into a separate file in split mode. - SplitDebuginfo::Unpacked => { - if cgcx.target_can_use_split_dwarf { - Some(dwo_out.as_path()) - } else { - None - } - } + let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { + // Don't change how DWARF is emitted when disabled. + (SplitDebuginfo::Off, _) => None, + // Don't provide a DWARF object path if split debuginfo is enabled but this is + // a platform that doesn't support Split DWARF. + _ if !cgcx.target_can_use_split_dwarf => None, + // Don't provide a DWARF object path in single mode, sections will be written + // into the object as normal but ignored by linker. + (_, SplitDwarfKind::Single) => None, + // Emit (a subset of the) DWARF into a separate dwarf object file in split + // mode. + (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), }; with_codegen(tm, llmod, config.no_builtins, |cpm| { @@ -947,7 +944,9 @@ pub(crate) unsafe fn codegen( Ok(module.into_compiled_module( config.emit_obj != EmitObj::None, - cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked, + cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split, config.emit_bc, &cgcx.output_filenames, )) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 8c3054b23ff..5217fa2758f 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1201,8 +1201,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn do_not_inline(&mut self, llret: &'ll Value) { - llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + // Cleanup is always the cold path. + llvm::Attribute::Cold.apply_callsite(llvm::AttributePlace::Function, llret); + + // In LLVM versions with deferred inlining (currently, system LLVM < 14), + // inlining drop glue can lead to exponential size blowup, see #41696 and #92110. + if !llvm_util::is_rust_llvm() && llvm_util::get_version() < (14, 0, 0) { + llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); + } } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 9f24a95482c..bb16bc5dccd 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -21,7 +21,7 @@ use rustc_middle::ty::layout::{ }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; -use rustc_session::config::{CFGuard, CrateType, DebugInfo}; +use rustc_session::config::{BranchProtection, CFGuard, CrateType, DebugInfo, PAuthKey, PacRet}; use rustc_session::Session; use rustc_span::source_map::Span; use rustc_span::symbol::Symbol; @@ -242,6 +242,34 @@ pub unsafe fn create_module<'ll>( } } + if sess.target.arch == "aarch64" { + let BranchProtection { bti, pac_ret: pac } = sess.opts.debugging_opts.branch_protection; + + llvm::LLVMRustAddModuleFlag( + llmod, + "branch-target-enforcement\0".as_ptr().cast(), + bti.into(), + ); + + llvm::LLVMRustAddModuleFlag( + llmod, + "sign-return-address\0".as_ptr().cast(), + pac.is_some().into(), + ); + let pac_opts = pac.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); + llvm::LLVMRustAddModuleFlag( + llmod, + "sign-return-address-all\0".as_ptr().cast(), + pac_opts.leaf.into(), + ); + let is_bkey = if pac_opts.key == PAuthKey::A { false } else { true }; + llvm::LLVMRustAddModuleFlag( + llmod, + "sign-return-address-with-bkey\0".as_ptr().cast(), + is_bkey.into(), + ); + } + llmod } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5f9c4189168..60ff18af0a9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -18,6 +18,7 @@ use crate::llvm::debuginfo::{ use crate::value::Value; use cstr::cstr; +use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; use rustc_codegen_ssa::traits::*; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; @@ -933,16 +934,16 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l // When targeting MSVC, emit MSVC style type names for compatibility with // .natvis visualizers (and perhaps other existing native debuggers?) - let msvc_like_names = cx.tcx.sess.target.is_like_msvc; + let cpp_like_debuginfo = cpp_like_debuginfo(cx.tcx); let (name, encoding) = match t.kind() { ty::Never => ("!", DW_ATE_unsigned), ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), ty::Bool => ("bool", DW_ATE_boolean), ty::Char => ("char", DW_ATE_unsigned_char), - ty::Int(int_ty) if msvc_like_names => (int_ty.msvc_basic_name(), DW_ATE_signed), - ty::Uint(uint_ty) if msvc_like_names => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), - ty::Float(float_ty) if msvc_like_names => (float_ty.msvc_basic_name(), DW_ATE_float), + ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), + ty::Uint(uint_ty) if cpp_like_debuginfo => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), + ty::Float(float_ty) if cpp_like_debuginfo => (float_ty.msvc_basic_name(), DW_ATE_float), ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), @@ -959,7 +960,7 @@ fn basic_type_metadata<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'l ) }; - if !msvc_like_names { + if !cpp_like_debuginfo { return ty_metadata; } @@ -1072,7 +1073,11 @@ pub fn compile_unit_metadata<'ll, 'tcx>( let output_filenames = tcx.output_filenames(()); let split_name = if tcx.sess.target_can_use_split_dwarf() { output_filenames - .split_dwarf_path(tcx.sess.split_debuginfo(), Some(codegen_unit_name)) + .split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.debugging_opts.split_dwarf_kind, + Some(codegen_unit_name), + ) // We get a path relative to the working directory from split_dwarf_path .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) } else { @@ -1521,13 +1526,6 @@ fn prepare_union_metadata<'ll, 'tcx>( // Enums //=----------------------------------------------------------------------------- -/// DWARF variant support is only available starting in LLVM 8, but -/// on MSVC we have to use the fallback mode, because LLVM doesn't -/// lower variant parts to PDB. -fn use_enum_fallback(cx: &CodegenCx<'_, '_>) -> bool { - cx.sess().target.is_like_msvc -} - // FIXME(eddyb) maybe precompute this? Right now it's computed once // per generator monomorphization, but it doesn't depend on substs. fn generator_layout_and_saved_local_names<'tcx>( @@ -1602,7 +1600,10 @@ impl<'ll, 'tcx> EnumMemberDescriptionFactory<'ll, 'tcx> { _ => bug!(), }; - let fallback = use_enum_fallback(cx); + // While LLVM supports generating debuginfo for variant types (enums), it doesn't support + // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting + // msvc, then we need to use a different, fallback encoding of the debuginfo. + let fallback = cpp_like_debuginfo(cx.tcx); // This will always find the metadata in the type map. let self_metadata = type_metadata(cx, self.enum_type, self.span); @@ -2155,7 +2156,10 @@ fn prepare_enum_metadata<'ll, 'tcx>( return FinalMetadata(discriminant_type_metadata(tag.value)); } - if use_enum_fallback(cx) { + // While LLVM supports generating debuginfo for variant types (enums), it doesn't support + // lowering that debuginfo to CodeView records for msvc targets. So if we are targeting + // msvc, then we need to use a different encoding of the debuginfo. + if cpp_like_debuginfo(tcx) { let discriminant_type_metadata = match layout.variants { Variants::Single { .. } => None, Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 3e7371179cc..01f7868df34 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -160,17 +160,17 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { // the values should match the ones in the DWARF standard anyway. let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; - let mut addr_ops = SmallVec::<[_; 8]>::new(); + let mut addr_ops = SmallVec::<[u64; 8]>::new(); if direct_offset.bytes() > 0 { addr_ops.push(op_plus_uconst()); - addr_ops.push(direct_offset.bytes() as i64); + addr_ops.push(direct_offset.bytes() as u64); } for &offset in indirect_offsets { addr_ops.push(op_deref()); if offset.bytes() > 0 { addr_ops.push(op_plus_uconst()); - addr_ops.push(offset.bytes() as i64); + addr_ops.push(offset.bytes() as u64); } } diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 90d0d5caba1..a6e06ffa819 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -47,6 +47,7 @@ fn declare_raw_fn<'ll>( attributes::default_optimisation_attrs(cx.tcx.sess, llfn); attributes::non_lazy_bind(cx.sess(), llfn); + llfn } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 6c911938ccc..f2782f84f55 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1902,6 +1902,8 @@ extern "C" { pub fn LLVMRustVersionMinor() -> u32; pub fn LLVMRustVersionPatch() -> u32; + pub fn LLVMRustIsRustLLVM() -> bool; + pub fn LLVMRustAddModuleFlag(M: &Module, name: *const c_char, value: u32); pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; @@ -2106,7 +2108,7 @@ extern "C" { Builder: &DIBuilder<'a>, Val: &'a Value, VarInfo: &'a DIVariable, - AddrOps: *const i64, + AddrOps: *const u64, AddrOpsCount: c_uint, DL: &'a DILocation, InsertAtEnd: &'a BasicBlock, @@ -2197,8 +2199,8 @@ extern "C" { Scope: &'a DIScope, InlinedAt: Option<&'a DILocation>, ) -> &'a DILocation; - pub fn LLVMRustDIBuilderCreateOpDeref() -> i64; - pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> i64; + pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; + pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; #[allow(improper_ctypes)] pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); @@ -2313,6 +2315,8 @@ extern "C" { end_callback: SelfProfileAfterPassCallback, ExtraPasses: *const c_char, ExtraPassesLen: size_t, + LLVMPlugins: *const c_char, + LLVMPluginsLen: size_t, ) -> LLVMRustResult; pub fn LLVMRustPrintModule( M: &Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index e4935ac431c..d49df29f453 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -121,14 +121,20 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMInitializePasses(); - // Register LLVM plugins by loading them into the compiler process. - for plugin in &sess.opts.debugging_opts.llvm_plugins { - let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); - debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); - - // Intentionally leak the dynamic library. We can't ever unload it - // since the library can make things that will live arbitrarily long. - mem::forget(lib); + // Use the legacy plugin registration if we don't use the new pass manager + if !should_use_new_llvm_pass_manager( + &sess.opts.debugging_opts.new_llvm_pass_manager, + &sess.target.arch, + ) { + // Register LLVM plugins by loading them into the compiler process. + for plugin in &sess.opts.debugging_opts.llvm_plugins { + let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); + debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); + + // Intentionally leak the dynamic library. We can't ever unload it + // since the library can make things that will live arbitrarily long. + mem::forget(lib); + } } rustc_llvm::initialize_available_targets(); @@ -217,6 +223,12 @@ pub fn get_version() -> (u32, u32, u32) { } } +/// Returns `true` if this LLVM is Rust's bundled LLVM (and not system LLVM). +pub fn is_rust_llvm() -> bool { + // Can be called without initializing LLVM + unsafe { llvm::LLVMRustIsRustLLVM() } +} + pub fn print_passes() { // Can be called without initializing LLVM unsafe { @@ -411,3 +423,13 @@ pub fn tune_cpu(sess: &Session) -> Option<&str> { let name = sess.opts.debugging_opts.tune_cpu.as_ref()?; Some(handle_native(name)) } + +pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_arch: &str) -> bool { + // The new pass manager is enabled by default for LLVM >= 13. + // This matches Clang, which also enables it since Clang 13. + + // FIXME: There are some perf issues with the new pass manager + // when targeting s390x, so it is temporarily disabled for that + // arch, see https://github.com/rust-lang/rust/issues/89609 + user_opt.unwrap_or_else(|| target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0)) +} |