From bcfc9b5073a92bbb4b1e4db2eab535357d8973ad Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Thu, 21 Aug 2025 14:32:10 +0100 Subject: inline at the callsite & warn when target features mismatch Co-authored-by: Jamie Cunliffe --- compiler/rustc_codegen_ssa/src/codegen_attrs.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 961bb788149..008340e614d 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -428,9 +428,16 @@ fn check_result( // llvm/llvm-project#70563). if !codegen_fn_attrs.target_features.is_empty() && matches!(codegen_fn_attrs.inline, InlineAttr::Always) + && !tcx.features().target_feature_inline_always() && let Some(span) = interesting_spans.inline { - tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`"); + feature_err( + tcx.sess, + sym::target_feature_inline_always, + span, + "cannot use `#[inline(always)]` with `#[target_feature]`", + ) + .emit(); } // warn that inline has no effect when no_sanitize is present -- cgit 1.4.1-3-g733a5 From fc7de9979e418b0ea467bb2833b07486d7c54940 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:36:10 +0000 Subject: Ensure we emit an allocator shim when only some crate types need one --- compiler/rustc_codegen_ssa/src/back/link.rs | 17 +++++++++++++---- compiler/rustc_codegen_ssa/src/base.rs | 27 ++++++++++++++++++++++----- tests/ui/linking/mixed-allocator-shim.rs | 15 +++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 tests/ui/linking/mixed-allocator-shim.rs (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 19c919c0e4e..48b01ea2df1 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -58,6 +58,7 @@ use super::linker::{self, Linker}; use super::metadata::{MetadataPosition, create_wrapper_file}; use super::rpath::{self, RPathConfig}; use super::{apple, versioned_llvm_target}; +use crate::base::needs_allocator_shim_for_linking; use crate::{ CodegenResults, CompiledModule, CrateInfo, NativeLib, errors, looks_like_rust_object_file, }; @@ -2080,9 +2081,17 @@ fn add_local_crate_regular_objects(cmd: &mut dyn Linker, codegen_results: &Codeg } /// Add object files for allocator code linked once for the whole crate tree. -fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { - if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) { - cmd.add_object(obj); +fn add_local_crate_allocator_objects( + cmd: &mut dyn Linker, + codegen_results: &CodegenResults, + crate_type: CrateType, +) { + if needs_allocator_shim_for_linking(&codegen_results.crate_info.dependency_formats, crate_type) + { + if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) + { + cmd.add_object(obj); + } } } @@ -2281,7 +2290,7 @@ fn linker_with_args( codegen_results, metadata, ); - add_local_crate_allocator_objects(cmd, codegen_results); + add_local_crate_allocator_objects(cmd, codegen_results, crate_type); // Avoid linking to dynamic libraries unless they satisfy some undefined symbols // at the point at which they are specified on the command line. diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 8abaf201aba..97cdf8b6973 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -17,6 +17,7 @@ use rustc_hir::lang_items::LangItem; use rustc_hir::{ItemId, Target}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType}; +use rustc_middle::middle::dependency_format::Dependencies; use rustc_middle::middle::exported_symbols::{self, SymbolExportKind}; use rustc_middle::middle::lang_items; use rustc_middle::mir::BinOp; @@ -630,14 +631,30 @@ pub fn allocator_kind_for_codegen(tcx: TyCtxt<'_>) -> Option { // If the crate doesn't have an `allocator_kind` set then there's definitely // no shim to generate. Otherwise we also check our dependency graph for all // our output crate types. If anything there looks like its a `Dynamic` - // linkage, then it's already got an allocator shim and we'll be using that - // one instead. If nothing exists then it's our job to generate the - // allocator! - let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { + // linkage for all crate types we may link as, then it's already got an + // allocator shim and we'll be using that one instead. If nothing exists + // then it's our job to generate the allocator! If crate types disagree + // about whether an allocator shim is necessary or not, we generate one + // and let needs_allocator_shim_for_linking decide at link time whether or + // not to use it for any particular linker invocation. + let all_crate_types_any_dynamic_crate = tcx.dependency_formats(()).iter().all(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; list.iter().any(|&linkage| linkage == Linkage::Dynamic) }); - if any_dynamic_crate { None } else { tcx.allocator_kind(()) } + if all_crate_types_any_dynamic_crate { None } else { tcx.allocator_kind(()) } +} + +/// Decide if this particular crate type needs an allocator shim linked in. +/// This may return true even when allocator_kind_for_codegen returns false. In +/// this case no allocator shim shall be linked. +pub(crate) fn needs_allocator_shim_for_linking( + dependency_formats: &Dependencies, + crate_type: CrateType, +) -> bool { + use rustc_middle::middle::dependency_format::Linkage; + let any_dynamic_crate = + dependency_formats[&crate_type].iter().any(|&linkage| linkage == Linkage::Dynamic); + !any_dynamic_crate } pub fn codegen_crate( diff --git a/tests/ui/linking/mixed-allocator-shim.rs b/tests/ui/linking/mixed-allocator-shim.rs new file mode 100644 index 00000000000..b81fbdc5205 --- /dev/null +++ b/tests/ui/linking/mixed-allocator-shim.rs @@ -0,0 +1,15 @@ +//@ build-pass +//@ compile-flags: --crate-type staticlib,dylib -Zstaticlib-prefer-dynamic +//@ no-prefer-dynamic + +// Test that compiling for multiple crate types in a single compilation with +// mismatching allocator shim requirements doesn't result in the allocator shim +// missing entirely. +// In this particular test the dylib crate type will statically link libstd and +// thus need an allocator shim, while the staticlib crate type will dynamically +// link libstd and thus not need an allocator shim. +// The -Zstaticlib-prefer-dynamic flag could be avoided by doing it the other +// way around, but testing that the staticlib correctly has the allocator shim +// in that case would require a run-make test instead. + +pub fn foo() {} -- cgit 1.4.1-3-g733a5 From 1d309008306f6695bdbbd535245067202ab44802 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 15:16:48 +0000 Subject: Fix typo in comment --- compiler/rustc_codegen_ssa/src/back/linker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index df1e91b12f9..03c30dc0c59 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -1827,7 +1827,7 @@ fn exported_symbols_for_non_proc_macro( let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { // Do not export mangled symbols from cdylibs and don't attempt to export compiler-builtins - // from any cdylib. The latter doesn't work anyway as we use hidden visibility for + // from any dylib. The latter doesn't work anyway as we use hidden visibility for // compiler-builtins. Most linkers silently ignore it, but ld64 gives a warning. if info.level.is_below_threshold(export_threshold) && !tcx.is_compiler_builtins(cnum) { symbols.push(( -- cgit 1.4.1-3-g733a5 From f4888c2a680aff0bd191704a8d44716eee94103e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 16:59:22 +0000 Subject: Correctly handle different crate types disagreeing if the allocator shim is exported Previously it would attempt to export the allocator shim even linking for a crate type which pulls in the allocator shim from a dylib rather than locally defining it. --- compiler/rustc_codegen_ssa/src/back/linker.rs | 15 ++++++- compiler/rustc_codegen_ssa/src/back/lto.rs | 8 +++- .../rustc_codegen_ssa/src/back/symbol_export.rs | 52 +++++++++++----------- 3 files changed, 46 insertions(+), 29 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 03c30dc0c59..a2efd420a32 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -11,8 +11,9 @@ use rustc_metadata::{ }; use rustc_middle::bug; use rustc_middle::middle::dependency_format::Linkage; -use rustc_middle::middle::exported_symbols; -use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; +use rustc_middle::middle::exported_symbols::{ + self, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, +}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; @@ -22,6 +23,8 @@ use tracing::{debug, warn}; use super::command::Command; use super::symbol_export; +use crate::back::symbol_export::allocator_shim_symbols; +use crate::base::needs_allocator_shim_for_linking; use crate::errors; #[cfg(test)] @@ -1838,6 +1841,14 @@ fn exported_symbols_for_non_proc_macro( } }); + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust + && needs_allocator_shim_for_linking(tcx.dependency_formats(()), crate_type) + && tcx.allocator_kind(()).is_some() + { + symbols.extend(allocator_shim_symbols(tcx)); + } + symbols } diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index c95038375a1..e6df6a2469f 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -8,8 +8,9 @@ use rustc_middle::ty::TyCtxt; use rustc_session::config::{CrateType, Lto}; use tracing::info; -use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate}; +use crate::back::symbol_export::{self, allocator_shim_symbols, symbol_name_for_instance_in_crate}; use crate::back::write::CodegenContext; +use crate::base::allocator_kind_for_codegen; use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro}; use crate::traits::*; @@ -115,6 +116,11 @@ pub(super) fn exported_symbols_for_lto( } } + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust && allocator_kind_for_codegen(tcx).is_some() { + symbols_below_threshold.extend(allocator_shim_symbols(tcx).map(|(name, _kind)| name)); + } + symbols_below_threshold } diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index d8a1480e911..b49e67217fb 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -18,7 +18,7 @@ use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::TlsModel; use tracing::debug; -use crate::base::allocator_kind_for_codegen; +use crate::back::symbol_export; fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { crates_export_threshold(tcx.crate_types()) @@ -217,31 +217,6 @@ fn exported_non_generic_symbols_provider_local<'tcx>( )); } - // Mark allocator shim symbols as exported only if they were generated. - if allocator_kind_for_codegen(tcx).is_some() { - for symbol_name in ALLOCATOR_METHODS - .iter() - .map(|method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) - .chain([ - mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), - ]) - { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - rustc_std_internal_symbol: true, - }, - )); - } - } - // Sort so we get a stable incr. comp. hash. symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); @@ -516,6 +491,31 @@ pub(crate) fn provide(providers: &mut Providers) { upstream_monomorphizations_for_provider; } +pub(crate) fn allocator_shim_symbols( + tcx: TyCtxt<'_>, +) -> impl Iterator { + ALLOCATOR_METHODS + .iter() + .map(move |method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) + .chain([ + mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), + mangle_internal_symbol(tcx, OomStrategy::SYMBOL), + mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), + ]) + .map(move |symbol_name| { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + ( + symbol_export::exporting_symbol_name_for_instance_in_crate( + tcx, + exported_symbol, + LOCAL_CRATE, + ), + SymbolExportKind::Text, + ) + }) +} + fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel { // We export anything that's not mangled at the "C" layer as it probably has // to do with ABI concerns. We do not, however, apply such treatment to -- cgit 1.4.1-3-g733a5 From 7a01c7f676cbffba4f14540ae90e4fd82f7b0af1 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:01:40 +0000 Subject: Export __rdl_* symbols to the allocator shim when doing LTO --- compiler/rustc_codegen_ssa/src/back/lto.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index e6df6a2469f..f4a9037940a 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -1,6 +1,7 @@ use std::ffi::CString; use std::sync::Arc; +use rustc_ast::expand::allocator::AllocatorKind; use rustc_data_structures::memmap::Mmap; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel}; @@ -95,6 +96,19 @@ pub(super) fn exported_symbols_for_lto( .filter_map(|&(s, info): &(ExportedSymbol<'_>, SymbolExportInfo)| { if info.level.is_below_threshold(export_threshold) || info.used { Some(symbol_name_for_instance_in_crate(tcx, s, cnum)) + } else if export_threshold == SymbolExportLevel::C + && info.rustc_std_internal_symbol + && let Some(AllocatorKind::Default) = allocator_kind_for_codegen(tcx) + { + // Export the __rdl_* exports for usage by the allocator shim when not using + // #[global_allocator]. Most of the conditions above are only used to avoid + // unnecessary expensive symbol_name_for_instance_in_crate calls. + let sym = symbol_name_for_instance_in_crate(tcx, s, cnum); + if sym.contains("__rdl_") || sym.contains("__rg_oom") { + Some(sym) + } else { + None + } } else { None } -- cgit 1.4.1-3-g733a5 From eea81b5d752f5310fcbd5ddae722afc7de0b8fa1 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:54:42 +0000 Subject: Ensure the allocator shim never participates in LTO Making it participate in LTO would be incorrect if you compile a crate as both a dylib (which needs it) and rlib (which must not include it) in the same rustc invocation. With linker plugin LTO, the allocator shim will still participate in LTO as it is safe to do so in that case. --- compiler/rustc_codegen_ssa/src/back/write.rs | 20 +++++++++++--------- compiler/rustc_codegen_ssa/src/base.rs | 11 ++--------- 2 files changed, 13 insertions(+), 18 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 8586615f7c7..d6f289b3eee 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -809,19 +809,12 @@ pub(crate) fn compute_per_cgu_lto_type( sess_lto: &Lto, opts: &config::Options, sess_crate_types: &[CrateType], - module_kind: ModuleKind, ) -> ComputedLtoType { // If the linker does LTO, we don't have to do it. Note that we // keep doing full LTO, if it is requested, as not to break the // assumption that the output will be a single module. let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - let is_allocator = module_kind == ModuleKind::Allocator; - // We ignore a request for full crate graph LTO if the crate type // is only an rlib, as there is no full crate graph to process, // that'll happen later. @@ -833,7 +826,7 @@ pub(crate) fn compute_per_cgu_lto_type( let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]); match sess_lto { - Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, + Lto::ThinLocal if !linker_does_lto => ComputedLtoType::Thin, Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin, Lto::Fat if !is_rlib => ComputedLtoType::Fat, _ => ComputedLtoType::No, @@ -855,7 +848,16 @@ fn execute_optimize_work_item( // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind); + // When we're automatically doing ThinLTO for multi-codegen-unit + // builds we don't actually want to LTO the allocator modules if + // it shows up. This is due to various linker shenanigans that + // we'll encounter later. + if module.kind == ModuleKind::Allocator { + let module = B::codegen(cgcx, module, module_config); + return WorkItemResult::Finished(module); + } + + let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 97cdf8b6973..071bd09249a 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -46,9 +46,7 @@ use crate::meth::load_vtable; use crate::mir::operand::OperandValue; use crate::mir::place::PlaceRef; use crate::traits::*; -use crate::{ - CachedModuleCodegen, CodegenLintLevels, CrateInfo, ModuleCodegen, ModuleKind, errors, meth, mir, -}; +use crate::{CachedModuleCodegen, CodegenLintLevels, CrateInfo, ModuleCodegen, errors, meth, mir}; pub(crate) fn bin_op_to_icmp_predicate(op: BinOp, signed: bool) -> IntPredicate { match (op, signed) { @@ -1135,12 +1133,7 @@ pub fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> // We can re-use either the pre- or the post-thinlto state. If no LTO is // being performed then we can use post-LTO artifacts, otherwise we must // reuse pre-LTO artifacts - match compute_per_cgu_lto_type( - &tcx.sess.lto(), - &tcx.sess.opts, - tcx.crate_types(), - ModuleKind::Regular, - ) { + match compute_per_cgu_lto_type(&tcx.sess.lto(), &tcx.sess.opts, tcx.crate_types()) { ComputedLtoType::No => CguReuse::PostLto, _ => CguReuse::PreLto, } -- cgit 1.4.1-3-g733a5 From 319fe230f0d960b343be31a1182dc0f10753156c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 28 Aug 2025 10:26:29 +0000 Subject: Special case allocator module submission to avoid special casing it elsewhere A lot of places had special handling just in case they would get an allocator module even though most of these places could never get one or would have a trivial implementation for the allocator module. Moving all handling of the allocator module to a single place simplifies things a fair bit. --- compiler/rustc_codegen_cranelift/src/driver/aot.rs | 11 +- compiler/rustc_codegen_gcc/src/back/lto.rs | 2 +- compiler/rustc_codegen_llvm/src/back/lto.rs | 22 ++-- compiler/rustc_codegen_ssa/src/back/write.rs | 135 ++++++++------------- compiler/rustc_codegen_ssa/src/base.rs | 53 ++++---- compiler/rustc_codegen_ssa/src/lib.rs | 2 - 6 files changed, 79 insertions(+), 146 deletions(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 7e77781dc2f..c3adb5e767e 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -12,9 +12,7 @@ use cranelift_object::{ObjectBuilder, ObjectModule}; use rustc_codegen_ssa::assert_module_sources::CguReuse; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::base::determine_cgu_reuse; -use rustc_codegen_ssa::{ - CodegenResults, CompiledModule, CrateInfo, ModuleKind, errors as ssa_errors, -}; +use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, errors as ssa_errors}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; @@ -363,7 +361,6 @@ fn emit_cgu( invocation_temp, prof, product.object, - ModuleKind::Regular, name.clone(), producer, )?; @@ -372,7 +369,6 @@ fn emit_cgu( module_regular, module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { name: format!("{name}.asm"), - kind: ModuleKind::Regular, object: Some(global_asm_object_file), dwarf_object: None, bytecode: None, @@ -389,7 +385,6 @@ fn emit_module( invocation_temp: Option<&str>, prof: &SelfProfilerRef, mut object: cranelift_object::object::write::Object<'_>, - kind: ModuleKind, name: String, producer_str: &str, ) -> Result { @@ -430,7 +425,6 @@ fn emit_module( Ok(CompiledModule { name, - kind, object: Some(tmp_file), dwarf_object: None, bytecode: None, @@ -485,7 +479,6 @@ fn reuse_workproduct_for_cgu( Ok(ModuleCodegenResult { module_regular: CompiledModule { name: cgu.name().to_string(), - kind: ModuleKind::Regular, object: Some(obj_out_regular), dwarf_object: None, bytecode: None, @@ -495,7 +488,6 @@ fn reuse_workproduct_for_cgu( }, module_global_asm: source_file_global_asm.map(|source_file| CompiledModule { name: cgu.name().to_string(), - kind: ModuleKind::Regular, object: Some(obj_out_global_asm), dwarf_object: None, bytecode: None, @@ -651,7 +643,6 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { tcx.sess.invocation_temp.as_deref(), &tcx.sess.prof, product.object, - ModuleKind::Allocator, "allocator_shim".to_owned(), &crate::debuginfo::producer(tcx.sess), ) { diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index fcee6b6df62..9d8ce2383f2 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -204,7 +204,7 @@ fn fat_lto( let path = tmp_path.path().to_path_buf().join(&module.name); let path = path.to_str().expect("path"); let context = &module.module_llvm.context; - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // NOTE: we need to set the optimization level here in order for LTO to do its job. context.set_optimization_level(to_gcc_opt_level(config.opt_level)); context.add_command_line_option("-flto=auto"); diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index fc38c4f3e51..326b876e7e6 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -11,7 +11,7 @@ use object::{Object, ObjectSection}; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; +use rustc_codegen_ssa::{ModuleCodegen, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; use rustc_errors::DiagCtxtHandle; @@ -43,9 +43,7 @@ fn prepare_lto( .map(|symbol| CString::new(symbol.to_owned()).unwrap()) .collect::>(); - if cgcx.regular_module_config.instrument_coverage - || cgcx.regular_module_config.pgo_gen.enabled() - { + if cgcx.module_config.instrument_coverage || cgcx.module_config.pgo_gen.enabled() { // These are weak symbols that point to the profile version and the // profile name, which need to be treated as exported so LTO doesn't nix // them. @@ -55,15 +53,15 @@ fn prepare_lto( symbols_below_threshold.extend(PROFILER_WEAK_SYMBOLS.iter().map(|&sym| sym.to_owned())); } - if cgcx.regular_module_config.sanitizer.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer.contains(SanitizerSet::MEMORY) { let mut msan_weak_symbols = Vec::new(); // Similar to profiling, preserve weak msan symbol during LTO. - if cgcx.regular_module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { + if cgcx.module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { msan_weak_symbols.push(c"__msan_keep_going"); } - if cgcx.regular_module_config.sanitizer_memory_track_origins != 0 { + if cgcx.module_config.sanitizer_memory_track_origins != 0 { msan_weak_symbols.push(c"__msan_track_origins"); } @@ -227,15 +225,9 @@ fn fat_lto( // All the other modules will be serialized and reparsed into the new // context, so this hopefully avoids serializing and parsing the largest // codegen unit. - // - // Additionally use a regular module as the base here to ensure that various - // file copy operations in the backend work correctly. The only other kind - // of module here should be an allocator one, and if your crate is smaller - // than the allocator module then the size doesn't really matter anyway. let costliest_module = in_memory .iter() .enumerate() - .filter(|&(_, module)| module.kind == ModuleKind::Regular) .map(|(i, module)| { let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; (cost, i) @@ -583,7 +575,7 @@ pub(crate) fn run_pass_manager( thin: bool, ) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -745,7 +737,7 @@ pub(crate) fn optimize_thin_module( let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. - if cgcx.config(ModuleKind::Regular).embed_bitcode() { + if cgcx.module_config.embed_bitcode() { module.thin_lto_buffer = Some(thin_module.data().to_vec()); } { diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index d6f289b3eee..f637e7f58db 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -333,8 +333,7 @@ pub struct CodegenContext { pub crate_types: Vec, pub output_filenames: Arc, pub invocation_temp: Option, - pub regular_module_config: Arc, - pub allocator_module_config: Arc, + pub module_config: Arc, pub tm_factory: TargetMachineFactoryFn, pub msvc_imps_needed: bool, pub is_pe_coff: bool, @@ -372,13 +371,6 @@ impl CodegenContext { pub fn create_dcx(&self) -> DiagCtxt { DiagCtxt::new(Box::new(self.diag_emitter.clone())) } - - pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } - } } fn generate_thin_lto_work( @@ -442,6 +434,7 @@ pub(crate) fn start_async_codegen( backend: B, tcx: TyCtxt<'_>, target_cpu: String, + allocator_module: Option>, ) -> OngoingCodegen { let (coordinator_send, coordinator_receive) = channel(); @@ -465,6 +458,7 @@ pub(crate) fn start_async_codegen( coordinator_receive, Arc::new(regular_config), Arc::new(allocator_config), + allocator_module, coordinator_send.clone(), ); @@ -495,7 +489,7 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir"); - for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { + for module in &compiled_modules.modules { let mut files = Vec::new(); if let Some(object_file_path) = &module.object { files.push((OutputType::Object.extension(), object_file_path.as_path())); @@ -720,15 +714,6 @@ pub(crate) enum WorkItem { } impl WorkItem { - fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | WorkItem::FatLto { .. } | WorkItem::ThinLto(_) => { - ModuleKind::Regular - } - } - } - /// Generate a short description of this work item suitable for use as a thread name. fn short_description(&self) -> String { // `pthread_setname()` on *nix ignores anything beyond the first 15 @@ -836,32 +821,22 @@ pub(crate) fn compute_per_cgu_lto_type( fn execute_optimize_work_item( cgcx: &CodegenContext, mut module: ModuleCodegen, - module_config: &ModuleConfig, ) -> WorkItemResult { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - B::optimize(cgcx, dcx, &mut module, module_config); + B::optimize(cgcx, dcx, &mut module, &cgcx.module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - if module.kind == ModuleKind::Allocator { - let module = B::codegen(cgcx, module, module_config); - return WorkItemResult::Finished(module); - } - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. - let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { + let bitcode = if cgcx.module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { @@ -870,7 +845,7 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } ComputedLtoType::Thin => { @@ -901,7 +876,6 @@ fn execute_optimize_work_item( fn execute_copy_from_cache_work_item( cgcx: &CodegenContext, module: CachedModuleCodegen, - module_config: &ModuleConfig, ) -> WorkItemResult { let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); @@ -961,6 +935,7 @@ fn execute_copy_from_cache_work_item( } }; + let module_config = &cgcx.module_config; let should_emit_obj = module_config.emit_obj != EmitObj::None; let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly); let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly); @@ -973,7 +948,6 @@ fn execute_copy_from_cache_work_item( WorkItemResult::Finished(CompiledModule { links_from_incr_cache, name: module.name, - kind: ModuleKind::Regular, object, dwarf_object, bytecode, @@ -988,7 +962,6 @@ fn execute_fat_lto_work_item( each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, - module_config: &ModuleConfig, ) -> WorkItemResult { for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) @@ -1000,17 +973,16 @@ fn execute_fat_lto_work_item( each_linked_rlib_for_lto, needs_fat_lto, ); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } fn execute_thin_lto_work_item( cgcx: &CodegenContext, module: lto::ThinModule, - module_config: &ModuleConfig, ) -> WorkItemResult { let module = B::optimize_thin(cgcx, module); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } @@ -1095,6 +1067,7 @@ fn start_executing_work( coordinator_receive: Receiver>, regular_config: Arc, allocator_config: Arc, + allocator_module: Option>, tx_to_llvm_workers: Sender>, ) -> thread::JoinHandle> { let coordinator_send = tx_to_llvm_workers; @@ -1159,8 +1132,7 @@ fn start_executing_work( expanded_args: tcx.sess.expanded_args.clone(), diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), - regular_module_config: regular_config, - allocator_module_config: allocator_config, + module_config: regular_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, @@ -1175,6 +1147,11 @@ fn start_executing_work( invocation_temp: sess.invocation_temp.clone(), }; + let compiled_allocator_module = allocator_module.map(|mut allocator_module| { + B::optimize(&cgcx, tcx.sess.dcx(), &mut allocator_module, &allocator_config); + B::codegen(&cgcx, allocator_module, &allocator_config) + }); + // This is the "main loop" of parallel work happening for parallel codegen. // It's here that we manage parallelism, schedule work, and work with // messages coming from clients. @@ -1314,7 +1291,6 @@ fn start_executing_work( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. let mut compiled_modules = vec![]; - let mut compiled_allocator_module = None; let mut needs_fat_lto = Vec::new(); let mut needs_thin_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); @@ -1588,15 +1564,7 @@ fn start_executing_work( match result { Ok(WorkItemResult::Finished(compiled_module)) => { - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - } + compiled_modules.push(compiled_module); } Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => { assert!(!started_lto); @@ -1724,45 +1692,38 @@ fn spawn_work<'a, B: ExtraBackendMethods>( let cgcx = cgcx.clone(); B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { - let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - let module_config = cgcx.config(work.module_kind()); - - match work { - WorkItem::Optimize(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); - execute_optimize_work_item(&cgcx, m, module_config) - } - WorkItem::CopyPostLtoArtifacts(m) => { - let _timer = cgcx.prof.generic_activity_with_arg( - "codegen_copy_artifacts_from_incr_cache", - &*m.name, - ); - execute_copy_from_cache_work_item(&cgcx, m, module_config) - } - WorkItem::FatLto { - exported_symbols_for_lto, - each_linked_rlib_for_lto, + let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { + WorkItem::Optimize(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); + execute_optimize_work_item(&cgcx, m) + } + WorkItem::CopyPostLtoArtifacts(m) => { + let _timer = cgcx + .prof + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name); + execute_copy_from_cache_work_item(&cgcx, m) + } + WorkItem::FatLto { + exported_symbols_for_lto, + each_linked_rlib_for_lto, + needs_fat_lto, + import_only_modules, + } => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", "everything"); + execute_fat_lto_work_item( + &cgcx, + &exported_symbols_for_lto, + &each_linked_rlib_for_lto, needs_fat_lto, import_only_modules, - } => { - let _timer = cgcx - .prof - .generic_activity_with_arg("codegen_module_perform_lto", "everything"); - execute_fat_lto_work_item( - &cgcx, - &exported_symbols_for_lto, - &each_linked_rlib_for_lto, - needs_fat_lto, - import_only_modules, - module_config, - ) - } - WorkItem::ThinLto(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); - execute_thin_lto_work_item(&cgcx, m, module_config) - } + ) + } + WorkItem::ThinLto(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); + execute_thin_lto_work_item(&cgcx, m) } })); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 071bd09249a..a9a2ae1b3db 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -662,7 +662,7 @@ pub fn codegen_crate( ) -> OngoingCodegen { // Skip crate items and just output metadata in -Z no-codegen mode. if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu); + let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, None); ongoing_codegen.codegen_finished(tcx); @@ -693,7 +693,27 @@ pub fn codegen_crate( } } - let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu); + // Codegen an allocator shim, if necessary. + let allocator_module = if let Some(kind) = allocator_kind_for_codegen(tcx) { + let llmod_id = + cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); + + tcx.sess.time("write_allocator_module", || { + let module = backend.codegen_allocator( + tcx, + &llmod_id, + kind, + // If allocator_kind is Some then alloc_error_handler_kind must + // also be Some. + tcx.alloc_error_handler_kind(()).unwrap(), + ); + Some(ModuleCodegen::new_allocator(llmod_id, module)) + }) + } else { + None + }; + + let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu, allocator_module); // For better throughput during parallel processing by LLVM, we used to sort // CGUs largest to smallest. This would lead to better thread utilization @@ -810,35 +830,6 @@ pub fn codegen_crate( } } - // Codegen an allocator shim, if necessary. - // Do this last to ensure the LLVM_passes timer doesn't start while no CGUs have been codegened - // yet for the backend to optimize. - if let Some(kind) = allocator_kind_for_codegen(tcx) { - let llmod_id = - cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); - let module_llvm = tcx.sess.time("write_allocator_module", || { - backend.codegen_allocator( - tcx, - &llmod_id, - kind, - // If allocator_kind is Some then alloc_error_handler_kind must - // also be Some. - tcx.alloc_error_handler_kind(()).unwrap(), - ) - }); - - ongoing_codegen.wait_for_signal_to_codegen_item(); - ongoing_codegen.check_for_errors(tcx.sess); - - // These modules are generally cheap and won't throw off scheduling. - let cost = 0; - submit_codegened_module_to_llvm( - &ongoing_codegen.coordinator, - ModuleCodegen::new_allocator(llmod_id, module_llvm), - cost, - ); - } - ongoing_codegen.codegen_finished(tcx); // Since the main thread is sometimes blocked during codegen, we keep track diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index fe0500a5d4c..5b90ffaa056 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -120,7 +120,6 @@ impl ModuleCodegen { CompiledModule { name: self.name.clone(), - kind: self.kind, object, dwarf_object, bytecode, @@ -134,7 +133,6 @@ impl ModuleCodegen { #[derive(Debug, Encodable, Decodable)] pub struct CompiledModule { pub name: String, - pub kind: ModuleKind, pub object: Option, pub dwarf_object: Option, pub bytecode: Option, -- cgit 1.4.1-3-g733a5