diff options
Diffstat (limited to 'compiler/rustc_codegen_ssa/src/back')
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/back/link.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/back/linker.rs | 17 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/back/lto.rs | 22 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/back/symbol_export.rs | 52 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/back/write.rs | 137 |
5 files changed, 124 insertions, 121 deletions
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 19c919c0e4e..48b01ea2df1 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -58,6 +58,7 @@ use super::linker::{self, Linker}; use super::metadata::{MetadataPosition, create_wrapper_file}; use super::rpath::{self, RPathConfig}; use super::{apple, versioned_llvm_target}; +use crate::base::needs_allocator_shim_for_linking; use crate::{ CodegenResults, CompiledModule, CrateInfo, NativeLib, errors, looks_like_rust_object_file, }; @@ -2080,9 +2081,17 @@ fn add_local_crate_regular_objects(cmd: &mut dyn Linker, codegen_results: &Codeg } /// Add object files for allocator code linked once for the whole crate tree. -fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { - if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) { - cmd.add_object(obj); +fn add_local_crate_allocator_objects( + cmd: &mut dyn Linker, + codegen_results: &CodegenResults, + crate_type: CrateType, +) { + if needs_allocator_shim_for_linking(&codegen_results.crate_info.dependency_formats, crate_type) + { + if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) + { + cmd.add_object(obj); + } } } @@ -2281,7 +2290,7 @@ fn linker_with_args( codegen_results, metadata, ); - add_local_crate_allocator_objects(cmd, codegen_results); + add_local_crate_allocator_objects(cmd, codegen_results, crate_type); // Avoid linking to dynamic libraries unless they satisfy some undefined symbols // at the point at which they are specified on the command line. diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index df1e91b12f9..a2efd420a32 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -11,8 +11,9 @@ use rustc_metadata::{ }; use rustc_middle::bug; use rustc_middle::middle::dependency_format::Linkage; -use rustc_middle::middle::exported_symbols; -use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; +use rustc_middle::middle::exported_symbols::{ + self, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, +}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; @@ -22,6 +23,8 @@ use tracing::{debug, warn}; use super::command::Command; use super::symbol_export; +use crate::back::symbol_export::allocator_shim_symbols; +use crate::base::needs_allocator_shim_for_linking; use crate::errors; #[cfg(test)] @@ -1827,7 +1830,7 @@ fn exported_symbols_for_non_proc_macro( let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { // Do not export mangled symbols from cdylibs and don't attempt to export compiler-builtins - // from any cdylib. The latter doesn't work anyway as we use hidden visibility for + // from any dylib. The latter doesn't work anyway as we use hidden visibility for // compiler-builtins. Most linkers silently ignore it, but ld64 gives a warning. if info.level.is_below_threshold(export_threshold) && !tcx.is_compiler_builtins(cnum) { symbols.push(( @@ -1838,6 +1841,14 @@ fn exported_symbols_for_non_proc_macro( } }); + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust + && needs_allocator_shim_for_linking(tcx.dependency_formats(()), crate_type) + && tcx.allocator_kind(()).is_some() + { + symbols.extend(allocator_shim_symbols(tcx)); + } + symbols } diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index c95038375a1..f4a9037940a 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -1,6 +1,7 @@ use std::ffi::CString; use std::sync::Arc; +use rustc_ast::expand::allocator::AllocatorKind; use rustc_data_structures::memmap::Mmap; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel}; @@ -8,8 +9,9 @@ use rustc_middle::ty::TyCtxt; use rustc_session::config::{CrateType, Lto}; use tracing::info; -use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate}; +use crate::back::symbol_export::{self, allocator_shim_symbols, symbol_name_for_instance_in_crate}; use crate::back::write::CodegenContext; +use crate::base::allocator_kind_for_codegen; use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro}; use crate::traits::*; @@ -94,6 +96,19 @@ pub(super) fn exported_symbols_for_lto( .filter_map(|&(s, info): &(ExportedSymbol<'_>, SymbolExportInfo)| { if info.level.is_below_threshold(export_threshold) || info.used { Some(symbol_name_for_instance_in_crate(tcx, s, cnum)) + } else if export_threshold == SymbolExportLevel::C + && info.rustc_std_internal_symbol + && let Some(AllocatorKind::Default) = allocator_kind_for_codegen(tcx) + { + // Export the __rdl_* exports for usage by the allocator shim when not using + // #[global_allocator]. Most of the conditions above are only used to avoid + // unnecessary expensive symbol_name_for_instance_in_crate calls. + let sym = symbol_name_for_instance_in_crate(tcx, s, cnum); + if sym.contains("__rdl_") || sym.contains("__rg_oom") { + Some(sym) + } else { + None + } } else { None } @@ -115,6 +130,11 @@ pub(super) fn exported_symbols_for_lto( } } + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust && allocator_kind_for_codegen(tcx).is_some() { + symbols_below_threshold.extend(allocator_shim_symbols(tcx).map(|(name, _kind)| name)); + } + symbols_below_threshold } diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index d8a1480e911..b49e67217fb 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -18,7 +18,7 @@ use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::TlsModel; use tracing::debug; -use crate::base::allocator_kind_for_codegen; +use crate::back::symbol_export; fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { crates_export_threshold(tcx.crate_types()) @@ -217,31 +217,6 @@ fn exported_non_generic_symbols_provider_local<'tcx>( )); } - // Mark allocator shim symbols as exported only if they were generated. - if allocator_kind_for_codegen(tcx).is_some() { - for symbol_name in ALLOCATOR_METHODS - .iter() - .map(|method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) - .chain([ - mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), - ]) - { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - rustc_std_internal_symbol: true, - }, - )); - } - } - // Sort so we get a stable incr. comp. hash. symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); @@ -516,6 +491,31 @@ pub(crate) fn provide(providers: &mut Providers) { upstream_monomorphizations_for_provider; } +pub(crate) fn allocator_shim_symbols( + tcx: TyCtxt<'_>, +) -> impl Iterator<Item = (String, SymbolExportKind)> { + ALLOCATOR_METHODS + .iter() + .map(move |method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) + .chain([ + mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), + mangle_internal_symbol(tcx, OomStrategy::SYMBOL), + mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), + ]) + .map(move |symbol_name| { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + ( + symbol_export::exporting_symbol_name_for_instance_in_crate( + tcx, + exported_symbol, + LOCAL_CRATE, + ), + SymbolExportKind::Text, + ) + }) +} + fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel { // We export anything that's not mangled at the "C" layer as it probably has // to do with ABI concerns. We do not, however, apply such treatment to diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 8586615f7c7..f637e7f58db 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -333,8 +333,7 @@ pub struct CodegenContext<B: WriteBackendMethods> { pub crate_types: Vec<CrateType>, pub output_filenames: Arc<OutputFilenames>, pub invocation_temp: Option<String>, - pub regular_module_config: Arc<ModuleConfig>, - pub allocator_module_config: Arc<ModuleConfig>, + pub module_config: Arc<ModuleConfig>, pub tm_factory: TargetMachineFactoryFn<B>, pub msvc_imps_needed: bool, pub is_pe_coff: bool, @@ -372,13 +371,6 @@ impl<B: WriteBackendMethods> CodegenContext<B> { pub fn create_dcx(&self) -> DiagCtxt { DiagCtxt::new(Box::new(self.diag_emitter.clone())) } - - pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } - } } fn generate_thin_lto_work<B: ExtraBackendMethods>( @@ -442,6 +434,7 @@ pub(crate) fn start_async_codegen<B: ExtraBackendMethods>( backend: B, tcx: TyCtxt<'_>, target_cpu: String, + allocator_module: Option<ModuleCodegen<B::Module>>, ) -> OngoingCodegen<B> { let (coordinator_send, coordinator_receive) = channel(); @@ -465,6 +458,7 @@ pub(crate) fn start_async_codegen<B: ExtraBackendMethods>( coordinator_receive, Arc::new(regular_config), Arc::new(allocator_config), + allocator_module, coordinator_send.clone(), ); @@ -495,7 +489,7 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir"); - for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { + for module in &compiled_modules.modules { let mut files = Vec::new(); if let Some(object_file_path) = &module.object { files.push((OutputType::Object.extension(), object_file_path.as_path())); @@ -720,15 +714,6 @@ pub(crate) enum WorkItem<B: WriteBackendMethods> { } impl<B: WriteBackendMethods> WorkItem<B> { - fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | WorkItem::FatLto { .. } | WorkItem::ThinLto(_) => { - ModuleKind::Regular - } - } - } - /// Generate a short description of this work item suitable for use as a thread name. fn short_description(&self) -> String { // `pthread_setname()` on *nix ignores anything beyond the first 15 @@ -809,19 +794,12 @@ pub(crate) fn compute_per_cgu_lto_type( sess_lto: &Lto, opts: &config::Options, sess_crate_types: &[CrateType], - module_kind: ModuleKind, ) -> ComputedLtoType { // If the linker does LTO, we don't have to do it. Note that we // keep doing full LTO, if it is requested, as not to break the // assumption that the output will be a single module. let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - let is_allocator = module_kind == ModuleKind::Allocator; - // We ignore a request for full crate graph LTO if the crate type // is only an rlib, as there is no full crate graph to process, // that'll happen later. @@ -833,7 +811,7 @@ pub(crate) fn compute_per_cgu_lto_type( let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]); match sess_lto { - Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin, + Lto::ThinLocal if !linker_does_lto => ComputedLtoType::Thin, Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin, Lto::Fat if !is_rlib => ComputedLtoType::Fat, _ => ComputedLtoType::No, @@ -843,23 +821,22 @@ pub(crate) fn compute_per_cgu_lto_type( fn execute_optimize_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, mut module: ModuleCodegen<B::Module>, - module_config: &ModuleConfig, ) -> WorkItemResult<B> { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - B::optimize(cgcx, dcx, &mut module, module_config); + B::optimize(cgcx, dcx, &mut module, &cgcx.module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it // back to the coordinator thread for further LTO processing (which // has to wait for all the initial modules to be optimized). - let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind); + let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types); // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. - let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { + let bitcode = if cgcx.module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { @@ -868,7 +845,7 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } ComputedLtoType::Thin => { @@ -899,7 +876,6 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>( fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: CachedModuleCodegen, - module_config: &ModuleConfig, ) -> WorkItemResult<B> { let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); @@ -959,6 +935,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>( } }; + let module_config = &cgcx.module_config; let should_emit_obj = module_config.emit_obj != EmitObj::None; let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly); let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly); @@ -971,7 +948,6 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>( WorkItemResult::Finished(CompiledModule { links_from_incr_cache, name: module.name, - kind: ModuleKind::Regular, object, dwarf_object, bytecode, @@ -986,7 +962,6 @@ fn execute_fat_lto_work_item<B: ExtraBackendMethods>( each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec<FatLtoInput<B>>, import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>, - module_config: &ModuleConfig, ) -> WorkItemResult<B> { for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) @@ -998,17 +973,16 @@ fn execute_fat_lto_work_item<B: ExtraBackendMethods>( each_linked_rlib_for_lto, needs_fat_lto, ); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } fn execute_thin_lto_work_item<B: ExtraBackendMethods>( cgcx: &CodegenContext<B>, module: lto::ThinModule<B>, - module_config: &ModuleConfig, ) -> WorkItemResult<B> { let module = B::optimize_thin(cgcx, module); - let module = B::codegen(cgcx, module, module_config); + let module = B::codegen(cgcx, module, &cgcx.module_config); WorkItemResult::Finished(module) } @@ -1093,6 +1067,7 @@ fn start_executing_work<B: ExtraBackendMethods>( coordinator_receive: Receiver<Message<B>>, regular_config: Arc<ModuleConfig>, allocator_config: Arc<ModuleConfig>, + allocator_module: Option<ModuleCodegen<B::Module>>, tx_to_llvm_workers: Sender<Message<B>>, ) -> thread::JoinHandle<Result<CompiledModules, ()>> { let coordinator_send = tx_to_llvm_workers; @@ -1157,8 +1132,7 @@ fn start_executing_work<B: ExtraBackendMethods>( expanded_args: tcx.sess.expanded_args.clone(), diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), - regular_module_config: regular_config, - allocator_module_config: allocator_config, + module_config: regular_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, @@ -1173,6 +1147,11 @@ fn start_executing_work<B: ExtraBackendMethods>( invocation_temp: sess.invocation_temp.clone(), }; + let compiled_allocator_module = allocator_module.map(|mut allocator_module| { + B::optimize(&cgcx, tcx.sess.dcx(), &mut allocator_module, &allocator_config); + B::codegen(&cgcx, allocator_module, &allocator_config) + }); + // This is the "main loop" of parallel work happening for parallel codegen. // It's here that we manage parallelism, schedule work, and work with // messages coming from clients. @@ -1312,7 +1291,6 @@ fn start_executing_work<B: ExtraBackendMethods>( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. let mut compiled_modules = vec![]; - let mut compiled_allocator_module = None; let mut needs_fat_lto = Vec::new(); let mut needs_thin_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); @@ -1586,15 +1564,7 @@ fn start_executing_work<B: ExtraBackendMethods>( match result { Ok(WorkItemResult::Finished(compiled_module)) => { - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - } + compiled_modules.push(compiled_module); } Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => { assert!(!started_lto); @@ -1722,45 +1692,38 @@ fn spawn_work<'a, B: ExtraBackendMethods>( let cgcx = cgcx.clone(); B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { - let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - let module_config = cgcx.config(work.module_kind()); - - match work { - WorkItem::Optimize(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); - execute_optimize_work_item(&cgcx, m, module_config) - } - WorkItem::CopyPostLtoArtifacts(m) => { - let _timer = cgcx.prof.generic_activity_with_arg( - "codegen_copy_artifacts_from_incr_cache", - &*m.name, - ); - execute_copy_from_cache_work_item(&cgcx, m, module_config) - } - WorkItem::FatLto { - exported_symbols_for_lto, - each_linked_rlib_for_lto, + let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { + WorkItem::Optimize(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); + execute_optimize_work_item(&cgcx, m) + } + WorkItem::CopyPostLtoArtifacts(m) => { + let _timer = cgcx + .prof + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*m.name); + execute_copy_from_cache_work_item(&cgcx, m) + } + WorkItem::FatLto { + exported_symbols_for_lto, + each_linked_rlib_for_lto, + needs_fat_lto, + import_only_modules, + } => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", "everything"); + execute_fat_lto_work_item( + &cgcx, + &exported_symbols_for_lto, + &each_linked_rlib_for_lto, needs_fat_lto, import_only_modules, - } => { - let _timer = cgcx - .prof - .generic_activity_with_arg("codegen_module_perform_lto", "everything"); - execute_fat_lto_work_item( - &cgcx, - &exported_symbols_for_lto, - &each_linked_rlib_for_lto, - needs_fat_lto, - import_only_modules, - module_config, - ) - } - WorkItem::ThinLto(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); - execute_thin_lto_work_item(&cgcx, m, module_config) - } + ) + } + WorkItem::ThinLto(m) => { + let _timer = + cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); + execute_thin_lto_work_item(&cgcx, m) } })); |