diff options
Diffstat (limited to 'src/librustc_codegen_llvm')
| -rw-r--r-- | src/librustc_codegen_llvm/back/archive.rs | 4 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/back/link.rs | 199 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/back/lto.rs | 424 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/back/write.rs | 1870 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/base.rs | 8 | ||||
| -rw-r--r-- | src/librustc_codegen_llvm/lib.rs | 165 |
6 files changed, 319 insertions, 2351 deletions
diff --git a/src/librustc_codegen_llvm/back/archive.rs b/src/librustc_codegen_llvm/back/archive.rs index 54245a36017..76c50711639 100644 --- a/src/librustc_codegen_llvm/back/archive.rs +++ b/src/librustc_codegen_llvm/back/archive.rs @@ -18,6 +18,7 @@ use std::ptr; use std::str; use back::bytecode::RLIB_BYTECODE_EXTENSION; +use rustc_codegen_ssa::back::archive::find_library; use libc; use llvm::archive_ro::{ArchiveRO, Child}; use llvm::{self, ArchiveKind}; @@ -52,7 +53,6 @@ enum Addition { }, } - fn is_relevant_child(c: &Child) -> bool { match c.name() { Some(name) => !name.contains("SYMDEF"), @@ -107,7 +107,7 @@ impl<'a> ArchiveBuilder<'a> { /// Adds all of the contents of a native library to this archive. This will /// search in the relevant locations for a library named `name`. pub fn add_native_library(&mut self, name: &str) { - let location = ::rustc_codegen_utils::find_library(name, &self.config.lib_search_paths, + let location = find_library(name, &self.config.lib_search_paths, self.config.sess); self.add_archive(&location, |_| false).unwrap_or_else(|e| { self.config.sess.fatal(&format!("failed to add native library {}: {}", diff --git a/src/librustc_codegen_llvm/back/link.rs b/src/librustc_codegen_llvm/back/link.rs index e3143e89f79..20f05d11087 100644 --- a/src/librustc_codegen_llvm/back/link.rs +++ b/src/librustc_codegen_llvm/back/link.rs @@ -9,9 +9,12 @@ // except according to those terms. use back::wasm; -use cc::windows_registry; use super::archive::{ArchiveBuilder, ArchiveConfig}; use super::bytecode::RLIB_BYTECODE_EXTENSION; +use rustc_codegen_ssa::back::linker::Linker; +use rustc_codegen_ssa::back::link::{remove, ignored_for_lto, each_linked_rlib, linker_and_flavor, + get_linker}; +use rustc_codegen_ssa::back::command::Command; use super::rpath::RPathConfig; use super::rpath; use metadata::METADATA_FILENAME; @@ -20,18 +23,15 @@ use rustc::session::config::{RUST_CGU_EXT, Lto}; use rustc::session::filesearch; use rustc::session::search_paths::PathKind; use rustc::session::Session; -use rustc::middle::cstore::{NativeLibrary, LibSource, NativeLibraryKind}; +use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind}; use rustc::middle::dependency_format::Linkage; -use rustc_codegen_ssa::CrateInfo; -use CodegenResults; +use rustc_codegen_ssa::CodegenResults; use rustc::util::common::time; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc::hir::def_id::CrateNum; use tempfile::{Builder as TempFileBuilder, TempDir}; use rustc_target::spec::{PanicStrategy, RelroLevel, LinkerFlavor}; use rustc_data_structures::fx::FxHashSet; -use rustc_codegen_utils::linker::Linker; -use rustc_codegen_utils::command::Command; use context::get_reloc_model; use llvm; @@ -51,69 +51,6 @@ pub use rustc_codegen_utils::link::{find_crate_name, filename_for_input, default invalid_output_for_target, filename_for_metadata, out_filename, check_file_is_writeable}; -// The third parameter is for env vars, used on windows to set up the -// path for MSVC to find its DLLs, and gcc to find its bundled -// toolchain -pub fn get_linker(sess: &Session, linker: &Path, flavor: LinkerFlavor) -> (PathBuf, Command) { - let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe"); - - // If our linker looks like a batch script on Windows then to execute this - // we'll need to spawn `cmd` explicitly. This is primarily done to handle - // emscripten where the linker is `emcc.bat` and needs to be spawned as - // `cmd /c emcc.bat ...`. - // - // This worked historically but is needed manually since #42436 (regression - // was tagged as #42791) and some more info can be found on #44443 for - // emscripten itself. - let mut cmd = match linker.to_str() { - Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker), - _ => match flavor { - LinkerFlavor::Lld(f) => Command::lld(linker, f), - LinkerFlavor::Msvc - if sess.opts.cg.linker.is_none() && sess.target.target.options.linker.is_none() => - { - Command::new(msvc_tool.as_ref().map(|t| t.path()).unwrap_or(linker)) - }, - _ => Command::new(linker), - } - }; - - // The compiler's sysroot often has some bundled tools, so add it to the - // PATH for the child. - let mut new_path = sess.host_filesearch(PathKind::All) - .get_tools_search_paths(); - let mut msvc_changed_path = false; - if sess.target.target.options.is_like_msvc { - if let Some(ref tool) = msvc_tool { - cmd.args(tool.args()); - for &(ref k, ref v) in tool.env() { - if k == "PATH" { - new_path.extend(env::split_paths(v)); - msvc_changed_path = true; - } else { - cmd.env(k, v); - } - } - } - } - - if !msvc_changed_path { - if let Some(path) = env::var_os("PATH") { - new_path.extend(env::split_paths(&path)); - } - } - cmd.env("PATH", env::join_paths(new_path).unwrap()); - - (linker.to_path_buf(), cmd) -} - -pub fn remove(sess: &Session, path: &Path) { - if let Err(e) = fs::remove_file(path) { - sess.err(&format!("failed to remove {}: {}", - path.display(), - e)); - } -} /// Perform the linkage portion of the compilation phase. This will generate all /// of the requested outputs for this compilation session. @@ -215,60 +152,6 @@ fn preserve_objects_for_their_debuginfo(sess: &Session) -> bool { false } -pub(crate) fn each_linked_rlib(sess: &Session, - info: &CrateInfo, - f: &mut dyn FnMut(CrateNum, &Path)) -> Result<(), String> { - let crates = info.used_crates_static.iter(); - let fmts = sess.dependency_formats.borrow(); - let fmts = fmts.get(&config::CrateType::Executable) - .or_else(|| fmts.get(&config::CrateType::Staticlib)) - .or_else(|| fmts.get(&config::CrateType::Cdylib)) - .or_else(|| fmts.get(&config::CrateType::ProcMacro)); - let fmts = match fmts { - Some(f) => f, - None => return Err("could not find formats for rlibs".to_string()) - }; - for &(cnum, ref path) in crates { - match fmts.get(cnum.as_usize() - 1) { - Some(&Linkage::NotLinked) | - Some(&Linkage::IncludedFromDylib) => continue, - Some(_) => {} - None => return Err("could not find formats for rlibs".to_string()) - } - let name = &info.crate_name[&cnum]; - let path = match *path { - LibSource::Some(ref p) => p, - LibSource::MetadataOnly => { - return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file", - name)) - } - LibSource::None => { - return Err(format!("could not find rlib for: `{}`", name)) - } - }; - f(cnum, &path); - } - Ok(()) -} - -/// Returns a boolean indicating whether the specified crate should be ignored -/// during LTO. -/// -/// Crates ignored during LTO are not lumped together in the "massive object -/// file" that we create and are linked in their normal rlib states. See -/// comments below for what crates do not participate in LTO. -/// -/// It's unusual for a crate to not participate in LTO. Typically only -/// compiler-specific and unstable crates have a reason to not participate in -/// LTO. -pub(crate) fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool { - // If our target enables builtin function lowering in LLVM then the - // crates providing these functions don't participate in LTO (e.g. - // no_builtins or compiler builtins crates). - !sess.target.target.options.no_builtins && - (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum)) -} - fn link_binary_output(sess: &Session, codegen_results: &CodegenResults, crate_type: config::CrateType, @@ -353,8 +236,11 @@ fn archive_config<'a>(sess: &'a Session, /// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a /// directory being searched for `extern crate` (observing an incomplete file). /// The returned path is the temporary file containing the complete metadata. -fn emit_metadata<'a>(sess: &'a Session, codegen_results: &CodegenResults, tmpdir: &TempDir) - -> PathBuf { +fn emit_metadata<'a>( + sess: &'a Session, + codegen_results: &CodegenResults, + tmpdir: &TempDir +) -> PathBuf { let out_filename = tmpdir.path().join(METADATA_FILENAME); let result = fs::write(&out_filename, &codegen_results.metadata.raw_data); @@ -576,69 +462,6 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLibrary]) { } } -pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) { - fn infer_from( - sess: &Session, - linker: Option<PathBuf>, - flavor: Option<LinkerFlavor>, - ) -> Option<(PathBuf, LinkerFlavor)> { - match (linker, flavor) { - (Some(linker), Some(flavor)) => Some((linker, flavor)), - // only the linker flavor is known; use the default linker for the selected flavor - (None, Some(flavor)) => Some((PathBuf::from(match flavor { - LinkerFlavor::Em => if cfg!(windows) { "emcc.bat" } else { "emcc" }, - LinkerFlavor::Gcc => "cc", - LinkerFlavor::Ld => "ld", - LinkerFlavor::Msvc => "link.exe", - LinkerFlavor::Lld(_) => "lld", - }), flavor)), - (Some(linker), None) => { - let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| { - sess.fatal("couldn't extract file stem from specified linker"); - }).to_owned(); - - let flavor = if stem == "emcc" { - LinkerFlavor::Em - } else if stem == "gcc" || stem.ends_with("-gcc") { - LinkerFlavor::Gcc - } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") { - LinkerFlavor::Ld - } else if stem == "link" || stem == "lld-link" { - LinkerFlavor::Msvc - } else if stem == "lld" || stem == "rust-lld" { - LinkerFlavor::Lld(sess.target.target.options.lld_flavor) - } else { - // fall back to the value in the target spec - sess.target.target.linker_flavor - }; - - Some((linker, flavor)) - }, - (None, None) => None, - } - } - - // linker and linker flavor specified via command line have precedence over what the target - // specification specifies - if let Some(ret) = infer_from( - sess, - sess.opts.cg.linker.clone(), - sess.opts.debugging_opts.linker_flavor, - ) { - return ret; - } - - if let Some(ret) = infer_from( - sess, - sess.target.target.options.linker.clone().map(PathBuf::from), - Some(sess.target.target.linker_flavor), - ) { - return ret; - } - - bug!("Not enough information provided to determine how to invoke the linker"); -} - // Create a dynamic library or executable // // This will invoke the system linker/cc to create the resulting file. This diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 0f62ea6d357..a5f07e46e11 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -9,12 +9,14 @@ // except according to those terms. use back::bytecode::{DecodedBytecode, RLIB_BYTECODE_EXTENSION}; -use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext}; -use back::write::{self, DiagnosticHandlers, pre_lto_bitcode_filename}; +use rustc_codegen_ssa::back::symbol_export; +use rustc_codegen_ssa::back::write::{ModuleConfig, CodegenContext, pre_lto_bitcode_filename}; +use rustc_codegen_ssa::back::lto::{SerializedModule, LtoModuleCodegen, ThinShared, ThinModule}; +use rustc_codegen_ssa::interfaces::*; +use back::write::{self, DiagnosticHandlers, with_llvm_pmb, save_temp_bitcode, get_llvm_opt_level}; use errors::{FatalError, Handler}; use llvm::archive_ro::ArchiveRO; use llvm::{self, True, False}; -use memmap; use rustc::dep_graph::WorkProduct; use rustc::dep_graph::cgu_reuse_tracker::CguReuse; use rustc::hir::def_id::LOCAL_CRATE; @@ -22,9 +24,8 @@ use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; use rustc_data_structures::fx::FxHashMap; -use rustc_codegen_utils::symbol_export; use time_graph::Timeline; -use ModuleLlvm; +use {ModuleLlvm, LlvmCodegenBackend}; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; use libc; @@ -47,71 +48,16 @@ pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { } } -pub(crate) enum LtoModuleCodegen { - Fat { - module: Option<ModuleCodegen<ModuleLlvm>>, - _serialized_bitcode: Vec<SerializedModule>, - }, - - Thin(ThinModule), -} - -impl LtoModuleCodegen { - pub fn name(&self) -> &str { - match *self { - LtoModuleCodegen::Fat { .. } => "everything", - LtoModuleCodegen::Thin(ref m) => m.name(), - } - } - - /// Optimize this module within the given codegen context. - /// - /// This function is unsafe as it'll return a `ModuleCodegen` still - /// points to LLVM data structures owned by this `LtoModuleCodegen`. - /// It's intended that the module returned is immediately code generated and - /// dropped, and then this LTO module is dropped. - pub(crate) unsafe fn optimize(&mut self, - cgcx: &CodegenContext, - timeline: &mut Timeline) - -> Result<ModuleCodegen<ModuleLlvm>, FatalError> - { - match *self { - LtoModuleCodegen::Fat { ref mut module, .. } => { - let module = module.take().unwrap(); - { - let config = cgcx.config(module.kind); - let llmod = module.module_llvm.llmod(); - let tm = &*module.module_llvm.tm; - run_pass_manager(cgcx, tm, llmod, config, false); - timeline.record("fat-done"); - } - Ok(module) - } - LtoModuleCodegen::Thin(ref mut thin) => thin.optimize(cgcx, timeline), - } - } - - /// A "gauge" of how costly it is to optimize this module, used to sort - /// biggest modules first. - pub fn cost(&self) -> u64 { - match *self { - // Only one module with fat LTO, so the cost doesn't matter. - LtoModuleCodegen::Fat { .. } => 0, - LtoModuleCodegen::Thin(ref m) => m.cost(), - } - } -} - /// Performs LTO, which in the case of full LTO means merging all modules into /// a single one and returning it for further optimizing. For ThinLTO, it will /// do the global analysis necessary and return two lists, one of the modules /// the need optimization and another for modules that can simply be copied over /// from the incr. comp. cache. -pub(crate) fn run(cgcx: &CodegenContext, +pub(crate) fn run(cgcx: &CodegenContext<LlvmCodegenBackend>, modules: Vec<ModuleCodegen<ModuleLlvm>>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, + cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, timeline: &mut Timeline) - -> Result<(Vec<LtoModuleCodegen>, Vec<WorkProduct>), FatalError> + -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { let diag_handler = cgcx.create_diag_handler(); let export_threshold = match cgcx.lto { @@ -230,13 +176,13 @@ pub(crate) fn run(cgcx: &CodegenContext, } } -fn fat_lto(cgcx: &CodegenContext, +fn fat_lto(cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, mut modules: Vec<ModuleCodegen<ModuleLlvm>>, - mut serialized_modules: Vec<(SerializedModule, CString)>, + mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, symbol_white_list: &[*const libc::c_char], timeline: &mut Timeline) - -> Result<Vec<LtoModuleCodegen>, FatalError> + -> Result<Vec<LtoModuleCodegen<LlvmCodegenBackend>>, FatalError> { info!("going for a fat lto"); @@ -303,7 +249,7 @@ fn fat_lto(cgcx: &CodegenContext, serialized_bitcode.push(bc_decoded); } drop(linker); - cgcx.save_temp_bitcode(&module, "lto.input"); + save_temp_bitcode(&cgcx, &module, "lto.input"); // Internalize everything that *isn't* in our whitelist to help strip out // more modules and such @@ -312,14 +258,14 @@ fn fat_lto(cgcx: &CodegenContext, llvm::LLVMRustRunRestrictionPass(llmod, ptr as *const *const libc::c_char, symbol_white_list.len() as libc::size_t); - cgcx.save_temp_bitcode(&module, "lto.after-restriction"); + save_temp_bitcode(&cgcx, &module, "lto.after-restriction"); } if cgcx.no_landing_pads { unsafe { llvm::LLVMRustMarkAllFunctionsNounwind(llmod); } - cgcx.save_temp_bitcode(&module, "lto.after-nounwind"); + save_temp_bitcode(&cgcx, &module, "lto.after-nounwind"); } timeline.record("passes"); } @@ -386,14 +332,14 @@ impl Drop for Linker<'a> { /// calculating the *index* for ThinLTO. This index will then be shared amongst /// all of the `LtoModuleCodegen` units returned below and destroyed once /// they all go out of scope. -fn thin_lto(cgcx: &CodegenContext, +fn thin_lto(cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, modules: Vec<ModuleCodegen<ModuleLlvm>>, - serialized_modules: Vec<(SerializedModule, CString)>, - cached_modules: Vec<(SerializedModule, WorkProduct)>, + serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, + cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, symbol_white_list: &[*const libc::c_char], timeline: &mut Timeline) - -> Result<(Vec<LtoModuleCodegen>, Vec<WorkProduct>), FatalError> + -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> { unsafe { info!("going for that thin, thin LTO"); @@ -556,9 +502,8 @@ fn thin_lto(cgcx: &CodegenContext, } } -fn run_pass_manager(cgcx: &CodegenContext, - tm: &llvm::TargetMachine, - llmod: &llvm::Module, +pub(crate) fn run_pass_manager(cgcx: &CodegenContext<LlvmCodegenBackend>, + module: &ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, thin: bool) { // Now we have one massive module inside of llmod. Time to run the @@ -569,7 +514,7 @@ fn run_pass_manager(cgcx: &CodegenContext, debug!("running the pass manager"); unsafe { let pm = llvm::LLVMCreatePassManager(); - llvm::LLVMRustAddAnalysisPasses(tm, pm, llmod); + llvm::LLVMRustAddAnalysisPasses(module.module_llvm.tm, pm, module.module_llvm.llmod()); if config.verify_llvm_ir { let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr() as *const _); @@ -588,12 +533,13 @@ fn run_pass_manager(cgcx: &CodegenContext, // Note that in general this shouldn't matter too much as you typically // only turn on ThinLTO when you're compiling with optimizations // otherwise. - let opt_level = config.opt_level.unwrap_or(llvm::CodeGenOptLevel::None); + let opt_level = config.opt_level.map(get_llvm_opt_level) + .unwrap_or(llvm::CodeGenOptLevel::None); let opt_level = match opt_level { llvm::CodeGenOptLevel::None => llvm::CodeGenOptLevel::Less, level => level, }; - with_llvm_pmb(llmod, config, opt_level, false, &mut |b| { + with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { if thin { llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); } else { @@ -615,29 +561,14 @@ fn run_pass_manager(cgcx: &CodegenContext, llvm::LLVMRustAddPass(pm, pass.unwrap()); } - time_ext(cgcx.time_passes, None, "LTO passes", || llvm::LLVMRunPassManager(pm, llmod)); + time_ext(cgcx.time_passes, None, "LTO passes", || + llvm::LLVMRunPassManager(pm, module.module_llvm.llmod())); llvm::LLVMDisposePassManager(pm); } debug!("lto done"); } -pub enum SerializedModule { - Local(ModuleBuffer), - FromRlib(Vec<u8>), - FromUncompressedFile(memmap::Mmap), -} - -impl SerializedModule { - fn data(&self) -> &[u8] { - match *self { - SerializedModule::Local(ref m) => m.data(), - SerializedModule::FromRlib(ref m) => m, - SerializedModule::FromUncompressedFile(ref m) => m, - } - } -} - pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); unsafe impl Send for ModuleBuffer {} @@ -649,8 +580,10 @@ impl ModuleBuffer { llvm::LLVMRustModuleBufferCreate(m) }) } +} - pub fn data(&self) -> &[u8] { +impl ModuleBufferMethods for ModuleBuffer { + fn data(&self) -> &[u8] { unsafe { let ptr = llvm::LLVMRustModuleBufferPtr(self.0); let len = llvm::LLVMRustModuleBufferLen(self.0); @@ -665,19 +598,7 @@ impl Drop for ModuleBuffer { } } -pub struct ThinModule { - shared: Arc<ThinShared>, - idx: usize, -} - -struct ThinShared { - data: ThinData, - thin_buffers: Vec<ThinBuffer>, - serialized_modules: Vec<SerializedModule>, - module_names: Vec<CString>, -} - -struct ThinData(&'static mut llvm::ThinLTOData); +pub struct ThinData(&'static mut llvm::ThinLTOData); unsafe impl Send for ThinData {} unsafe impl Sync for ThinData {} @@ -702,8 +623,10 @@ impl ThinBuffer { ThinBuffer(buffer) } } +} - pub fn data(&self) -> &[u8] { +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { unsafe { let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; let len = llvm::LLVMRustThinLTOBufferLen(self.0); @@ -720,161 +643,142 @@ impl Drop for ThinBuffer { } } -impl ThinModule { - fn name(&self) -> &str { - self.shared.module_names[self.idx].to_str().unwrap() - } - - fn cost(&self) -> u64 { - // Yes, that's correct, we're using the size of the bytecode as an - // indicator for how costly this codegen unit is. - self.data().len() as u64 - } - - fn data(&self) -> &[u8] { - let a = self.shared.thin_buffers.get(self.idx).map(|b| b.data()); - a.unwrap_or_else(|| { - let len = self.shared.thin_buffers.len(); - self.shared.serialized_modules[self.idx - len].data() - }) - } - - unsafe fn optimize(&mut self, cgcx: &CodegenContext, timeline: &mut Timeline) - -> Result<ModuleCodegen<ModuleLlvm>, FatalError> - { - let diag_handler = cgcx.create_diag_handler(); - let tm = (cgcx.tm_factory)().map_err(|e| { - write::llvm_err(&diag_handler, &e) - })?; - - // Right now the implementation we've got only works over serialized - // modules, so we create a fresh new LLVM context and parse the module - // into that context. One day, however, we may do this for upstream - // crates but for locally codegened modules we may be able to reuse - // that LLVM Context and Module. - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = llvm::LLVMRustParseBitcodeForThinLTO( +pub unsafe fn optimize_thin_module( + thin_module: &mut ThinModule<LlvmCodegenBackend>, + cgcx: &CodegenContext<LlvmCodegenBackend>, + timeline: &mut Timeline +) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + let tm = (cgcx.tm_factory)().map_err(|e| { + write::llvm_err(&diag_handler, &e) + })?; + + // Right now the implementation we've got only works over serialized + // modules, so we create a fresh new LLVM context and parse the module + // into that context. One day, however, we may do this for upstream + // crates but for locally codegened modules we may be able to reuse + // that LLVM Context and Module. + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let llmod_raw = llvm::LLVMRustParseBitcodeForThinLTO( + llcx, + thin_module.data().as_ptr(), + thin_module.data().len(), + thin_module.shared.module_names[thin_module.idx].as_ptr(), + ).ok_or_else(|| { + let msg = "failed to parse bitcode for thin LTO module"; + write::llvm_err(&diag_handler, msg) + })? as *const _; + let module = ModuleCodegen { + module_llvm: ModuleLlvm { + llmod_raw, llcx, - self.data().as_ptr(), - self.data().len(), - self.shared.module_names[self.idx].as_ptr(), - ).ok_or_else(|| { - let msg = "failed to parse bitcode for thin LTO module"; - write::llvm_err(&diag_handler, msg) - })? as *const _; - let module = ModuleCodegen { - module_llvm: ModuleLlvm { - llmod_raw, - llcx, - tm, - }, - name: self.name().to_string(), - kind: ModuleKind::Regular, - }; - { - let llmod = module.module_llvm.llmod(); - cgcx.save_temp_bitcode(&module, "thin-lto-input"); - - // Before we do much else find the "main" `DICompileUnit` that we'll be - // using below. If we find more than one though then rustc has changed - // in a way we're not ready for, so generate an ICE by returning - // an error. - let mut cu1 = ptr::null_mut(); - let mut cu2 = ptr::null_mut(); - llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); - if !cu2.is_null() { - let msg = "multiple source DICompileUnits found"; - return Err(write::llvm_err(&diag_handler, msg)) - } + tm, + }, + name: thin_module.name().to_string(), + kind: ModuleKind::Regular, + }; + { + let llmod = module.module_llvm.llmod(); + save_temp_bitcode(&cgcx, &module, "thin-lto-input"); + + // Before we do much else find the "main" `DICompileUnit` that we'll be + // using below. If we find more than one though then rustc has changed + // in a way we're not ready for, so generate an ICE by returning + // an error. + let mut cu1 = ptr::null_mut(); + let mut cu2 = ptr::null_mut(); + llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + if !cu2.is_null() { + let msg = "multiple source DICompileUnits found"; + return Err(write::llvm_err(&diag_handler, msg)) + } - // Like with "fat" LTO, get some better optimizations if landing pads - // are disabled by removing all landing pads. - if cgcx.no_landing_pads { - llvm::LLVMRustMarkAllFunctionsNounwind(llmod); - cgcx.save_temp_bitcode(&module, "thin-lto-after-nounwind"); - timeline.record("nounwind"); - } + // Like with "fat" LTO, get some better optimizations if landing pads + // are disabled by removing all landing pads. + if cgcx.no_landing_pads { + llvm::LLVMRustMarkAllFunctionsNounwind(llmod); + save_temp_bitcode(&cgcx, &module, "thin-lto-after-nounwind"); + timeline.record("nounwind"); + } - // Up next comes the per-module local analyses that we do for Thin LTO. - // Each of these functions is basically copied from the LLVM - // implementation and then tailored to suit this implementation. Ideally - // each of these would be supported by upstream LLVM but that's perhaps - // a patch for another day! - // - // You can find some more comments about these functions in the LLVM - // bindings we've got (currently `PassWrapper.cpp`) - if !llvm::LLVMRustPrepareThinLTORename(self.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)) - } - cgcx.save_temp_bitcode(&module, "thin-lto-after-rename"); - timeline.record("rename"); - if !llvm::LLVMRustPrepareThinLTOResolveWeak(self.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)) - } - cgcx.save_temp_bitcode(&module, "thin-lto-after-resolve"); - timeline.record("resolve"); - if !llvm::LLVMRustPrepareThinLTOInternalize(self.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)) - } - cgcx.save_temp_bitcode(&module, "thin-lto-after-internalize"); - timeline.record("internalize"); - if !llvm::LLVMRustPrepareThinLTOImport(self.shared.data.0, llmod) { - let msg = "failed to prepare thin LTO module"; - return Err(write::llvm_err(&diag_handler, msg)) - } - cgcx.save_temp_bitcode(&module, "thin-lto-after-import"); - timeline.record("import"); - - // Ok now this is a bit unfortunate. This is also something you won't - // find upstream in LLVM's ThinLTO passes! This is a hack for now to - // work around bugs in LLVM. - // - // First discovered in #45511 it was found that as part of ThinLTO - // importing passes LLVM will import `DICompileUnit` metadata - // information across modules. This means that we'll be working with one - // LLVM module that has multiple `DICompileUnit` instances in it (a - // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of - // bugs in LLVM's backend which generates invalid DWARF in a situation - // like this: - // - // https://bugs.llvm.org/show_bug.cgi?id=35212 - // https://bugs.llvm.org/show_bug.cgi?id=35562 - // - // While the first bug there is fixed the second ended up causing #46346 - // which was basically a resurgence of #45511 after LLVM's bug 35212 was - // fixed. - // - // This function below is a huge hack around this problem. The function - // below is defined in `PassWrapper.cpp` and will basically "merge" - // all `DICompileUnit` instances in a module. Basically it'll take all - // the objects, rewrite all pointers of `DISubprogram` to point to the - // first `DICompileUnit`, and then delete all the other units. - // - // This is probably mangling to the debug info slightly (but hopefully - // not too much) but for now at least gets LLVM to emit valid DWARF (or - // so it appears). Hopefully we can remove this once upstream bugs are - // fixed in LLVM. - llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); - cgcx.save_temp_bitcode(&module, "thin-lto-after-patch"); - timeline.record("patch"); - - // Alright now that we've done everything related to the ThinLTO - // analysis it's time to run some optimizations! Here we use the same - // `run_pass_manager` as the "fat" LTO above except that we tell it to - // populate a thin-specific pass manager, which presumably LLVM treats a - // little differently. - info!("running thin lto passes over {}", module.name); - let config = cgcx.config(module.kind); - run_pass_manager(cgcx, module.module_llvm.tm, llmod, config, true); - cgcx.save_temp_bitcode(&module, "thin-lto-after-pm"); - timeline.record("thin-done"); + // Up next comes the per-module local analyses that we do for Thin LTO. + // Each of these functions is basically copied from the LLVM + // implementation and then tailored to suit this implementation. Ideally + // each of these would be supported by upstream LLVM but that's perhaps + // a patch for another day! + // + // You can find some more comments about these functions in the LLVM + // bindings we've got (currently `PassWrapper.cpp`) + if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)) } + save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); + timeline.record("rename"); + if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)) + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); + timeline.record("resolve"); + if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)) + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); + timeline.record("internalize"); + if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)) + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); + timeline.record("import"); - Ok(module) + // Ok now this is a bit unfortunate. This is also something you won't + // find upstream in LLVM's ThinLTO passes! This is a hack for now to + // work around bugs in LLVM. + // + // First discovered in #45511 it was found that as part of ThinLTO + // importing passes LLVM will import `DICompileUnit` metadata + // information across modules. This means that we'll be working with one + // LLVM module that has multiple `DICompileUnit` instances in it (a + // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of + // bugs in LLVM's backend which generates invalid DWARF in a situation + // like this: + // + // https://bugs.llvm.org/show_bug.cgi?id=35212 + // https://bugs.llvm.org/show_bug.cgi?id=35562 + // + // While the first bug there is fixed the second ended up causing #46346 + // which was basically a resurgence of #45511 after LLVM's bug 35212 was + // fixed. + // + // This function below is a huge hack around this problem. The function + // below is defined in `PassWrapper.cpp` and will basically "merge" + // all `DICompileUnit` instances in a module. Basically it'll take all + // the objects, rewrite all pointers of `DISubprogram` to point to the + // first `DICompileUnit`, and then delete all the other units. + // + // This is probably mangling to the debug info slightly (but hopefully + // not too much) but for now at least gets LLVM to emit valid DWARF (or + // so it appears). Hopefully we can remove this once upstream bugs are + // fixed in LLVM. + llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); + save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); + timeline.record("patch"); + + // Alright now that we've done everything related to the ThinLTO + // analysis it's time to run some optimizations! Here we use the same + // `run_pass_manager` as the "fat" LTO above except that we tell it to + // populate a thin-specific pass manager, which presumably LLVM treats a + // little differently. + info!("running thin lto passes over {}", module.name); + let config = cgcx.config(module.kind); + run_pass_manager(cgcx, &module, config, true); + save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); + timeline.record("thin-done"); } + Ok(module) } #[derive(Debug, Default)] diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs index 1e167046daf..97adfab516f 100644 --- a/src/librustc_codegen_llvm/back/write.rs +++ b/src/librustc_codegen_llvm/back/write.rs @@ -10,57 +10,35 @@ use attributes; use back::bytecode::{self, RLIB_BYTECODE_EXTENSION}; -use back::lto::{self, ThinBuffer, SerializedModule}; -use back::link::{self, get_linker, remove}; +use back::lto::ThinBuffer; +use rustc_codegen_ssa::back::write::{CodegenContext, ModuleConfig, run_assembler}; +use rustc_codegen_ssa::interfaces::*; use base; use consts; -use memmap; -use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir, - in_incr_comp_dir, in_incr_comp_dir_sess}; -use rustc::dep_graph::{WorkProduct, WorkProductId, WorkProductFileKind}; -use rustc::dep_graph::cgu_reuse_tracker::CguReuseTracker; -use rustc::middle::cstore::EncodedMetadata; -use rustc::session::config::{self, OutputFilenames, OutputType, Passes, Sanitizer, Lto}; +use rustc::session::config::{self, OutputType, Passes, Lto}; use rustc::session::Session; -use rustc::util::nodemap::FxHashMap; -use time_graph::{self, TimeGraph, Timeline}; +use time_graph::Timeline; use llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; use llvm_util; -use {CodegenResults, ModuleLlvm}; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, CachedModuleCodegen, CompiledModule, CrateInfo}; -use rustc::hir::def_id::{CrateNum, LOCAL_CRATE}; -use rustc::ty::TyCtxt; -use rustc::util::common::{time_ext, time_depth, set_time_depth, print_time_passes_entry}; +use ModuleLlvm; +use rustc_codegen_ssa::{ModuleCodegen, CompiledModule}; +use rustc::util::common::time_ext; use rustc_fs_util::{path2cstr, link_or_copy}; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_data_structures::svh::Svh; -use rustc_codegen_utils::command::Command; -use rustc_codegen_utils::linker::LinkerInfo; -use rustc_codegen_utils::symbol_export::ExportedSymbols; -use errors::{self, Handler, Level, DiagnosticBuilder, FatalError, DiagnosticId}; -use errors::emitter::{Emitter}; -use syntax::attr; -use syntax::ext::hygiene::Mark; -use syntax_pos::MultiSpan; -use syntax_pos::symbol::Symbol; +use errors::{self, Handler, FatalError}; use type_::Type; use context::{is_pie_binary, get_reloc_model}; use common; -use jobserver::{Client, Acquired}; +use LlvmCodegenBackend; use rustc_demangle; -use std::any::Any; use std::ffi::{CString, CStr}; use std::fs; use std::io::{self, Write}; -use std::mem; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::str; use std::sync::Arc; -use std::sync::mpsc::{channel, Sender, Receiver}; use std::slice; -use std::time::Instant; -use std::thread; use libc::{c_uint, c_void, c_char, size_t}; pub const RELOC_MODEL_ARGS : [(&str, llvm::RelocMode); 7] = [ @@ -87,8 +65,6 @@ pub const TLS_MODEL_ARGS : [(&str, llvm::ThreadLocalMode); 4] = [ ("local-exec", llvm::ThreadLocalMode::LocalExec), ]; -const PRE_THIN_LTO_BC_EXT: &str = "pre-thin-lto.bc"; - pub fn llvm_err(handler: &errors::Handler, msg: &str) -> FatalError { match llvm::last_error() { Some(err) => handler.fatal(&format!("{}: {}", msg, err)), @@ -115,7 +91,7 @@ pub fn write_output_file( } } -fn get_llvm_opt_level(optimize: config::OptLevel) -> llvm::CodeGenOptLevel { +pub(crate) fn get_llvm_opt_level(optimize: config::OptLevel) -> llvm::CodeGenOptLevel { match optimize { config::OptLevel::No => llvm::CodeGenOptLevel::None, config::OptLevel::Less => llvm::CodeGenOptLevel::Less, @@ -125,7 +101,7 @@ fn get_llvm_opt_level(optimize: config::OptLevel) -> llvm::CodeGenOptLevel { } } -fn get_llvm_opt_size(optimize: config::OptLevel) -> llvm::CodeGenOptSize { +pub(crate) fn get_llvm_opt_size(optimize: config::OptLevel) -> llvm::CodeGenOptSize { match optimize { config::OptLevel::Size => llvm::CodeGenOptSizeDefault, config::OptLevel::SizeMin => llvm::CodeGenOptSizeAggressive, @@ -223,212 +199,31 @@ pub fn target_machine_factory(sess: &Session, find_features: bool) }) } -/// Module-specific configuration for `optimize_and_codegen`. -pub struct ModuleConfig { - /// Names of additional optimization passes to run. - passes: Vec<String>, - /// Some(level) to optimize at a certain level, or None to run - /// absolutely no optimizations (used for the metadata module). - pub opt_level: Option<llvm::CodeGenOptLevel>, - - /// Some(level) to optimize binary size, or None to not affect program size. - opt_size: Option<llvm::CodeGenOptSize>, - - pgo_gen: Option<String>, - pgo_use: String, - - // Flags indicating which outputs to produce. - pub emit_pre_thin_lto_bc: bool, - emit_no_opt_bc: bool, - emit_bc: bool, - emit_bc_compressed: bool, - emit_lto_bc: bool, - emit_ir: bool, - emit_asm: bool, - emit_obj: bool, - // Miscellaneous flags. These are mostly copied from command-line - // options. - pub verify_llvm_ir: bool, - no_prepopulate_passes: bool, - no_builtins: bool, - time_passes: bool, - vectorize_loop: bool, - vectorize_slp: bool, - merge_functions: bool, - inline_threshold: Option<usize>, - // Instead of creating an object file by doing LLVM codegen, just - // make the object file bitcode. Provides easy compatibility with - // emscripten's ecc compiler, when used as the linker. - obj_is_bitcode: bool, - no_integrated_as: bool, - embed_bitcode: bool, - embed_bitcode_marker: bool, -} - -impl ModuleConfig { - fn new(passes: Vec<String>) -> ModuleConfig { - ModuleConfig { - passes, - opt_level: None, - opt_size: None, - - pgo_gen: None, - pgo_use: String::new(), - - emit_no_opt_bc: false, - emit_pre_thin_lto_bc: false, - emit_bc: false, - emit_bc_compressed: false, - emit_lto_bc: false, - emit_ir: false, - emit_asm: false, - emit_obj: false, - obj_is_bitcode: false, - embed_bitcode: false, - embed_bitcode_marker: false, - no_integrated_as: false, - - verify_llvm_ir: false, - no_prepopulate_passes: false, - no_builtins: false, - time_passes: false, - vectorize_loop: false, - vectorize_slp: false, - merge_functions: false, - inline_threshold: None - } - } - - fn set_flags(&mut self, sess: &Session, no_builtins: bool) { - self.verify_llvm_ir = sess.verify_llvm_ir(); - self.no_prepopulate_passes = sess.opts.cg.no_prepopulate_passes; - self.no_builtins = no_builtins || sess.target.target.options.no_builtins; - self.time_passes = sess.time_passes(); - self.inline_threshold = sess.opts.cg.inline_threshold; - self.obj_is_bitcode = sess.target.target.options.obj_is_bitcode || - sess.opts.debugging_opts.cross_lang_lto.enabled(); - let embed_bitcode = sess.target.target.options.embed_bitcode || - sess.opts.debugging_opts.embed_bitcode; - if embed_bitcode { - match sess.opts.optimize { - config::OptLevel::No | - config::OptLevel::Less => { - self.embed_bitcode_marker = embed_bitcode; - } - _ => self.embed_bitcode = embed_bitcode, - } - } - - // Copy what clang does by turning on loop vectorization at O2 and - // slp vectorization at O3. Otherwise configure other optimization aspects - // of this pass manager builder. - // Turn off vectorization for emscripten, as it's not very well supported. - self.vectorize_loop = !sess.opts.cg.no_vectorize_loops && - (sess.opts.optimize == config::OptLevel::Default || - sess.opts.optimize == config::OptLevel::Aggressive) && - !sess.target.target.options.is_like_emscripten; - - self.vectorize_slp = !sess.opts.cg.no_vectorize_slp && - sess.opts.optimize == config::OptLevel::Aggressive && - !sess.target.target.options.is_like_emscripten; - - self.merge_functions = sess.opts.optimize == config::OptLevel::Default || - sess.opts.optimize == config::OptLevel::Aggressive; - } - - pub fn bitcode_needed(&self) -> bool { - self.emit_bc || self.obj_is_bitcode - || self.emit_bc_compressed || self.embed_bitcode - } -} - -/// Assembler name and command used by codegen when no_integrated_as is enabled -struct AssemblerCommand { - name: PathBuf, - cmd: Command, -} - -/// Additional resources used by optimize_and_codegen (not module specific) -#[derive(Clone)] -pub struct CodegenContext { - // Resources needed when running LTO - pub time_passes: bool, - pub lto: Lto, - pub no_landing_pads: bool, - pub save_temps: bool, - pub fewer_names: bool, - pub exported_symbols: Option<Arc<ExportedSymbols>>, - pub opts: Arc<config::Options>, - pub crate_types: Vec<config::CrateType>, - pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>, - output_filenames: Arc<OutputFilenames>, - regular_module_config: Arc<ModuleConfig>, - metadata_module_config: Arc<ModuleConfig>, - allocator_module_config: Arc<ModuleConfig>, - pub tm_factory: Arc<dyn Fn() -> Result<&'static mut llvm::TargetMachine, String> + Send + Sync>, - pub msvc_imps_needed: bool, - pub target_pointer_width: String, - debuginfo: config::DebugInfo, - - // Number of cgus excluding the allocator/metadata modules - pub total_cgus: usize, - // Handler to use for diagnostics produced during codegen. - pub diag_emitter: SharedEmitter, - // LLVM passes added by plugins. - pub plugin_passes: Vec<String>, - // LLVM optimizations for which we want to print remarks. - pub remark: Passes, - // Worker thread number - pub worker: usize, - // The incremental compilation session directory, or None if we are not - // compiling incrementally - pub incr_comp_session_dir: Option<PathBuf>, - // Used to update CGU re-use information during the thinlto phase. - pub cgu_reuse_tracker: CguReuseTracker, - // Channel back to the main control thread to send messages to - coordinator_send: Sender<Box<dyn Any + Send>>, - // A reference to the TimeGraph so we can register timings. None means that - // measuring is disabled. - time_graph: Option<TimeGraph>, - // The assembler command if no_integrated_as option is enabled, None otherwise - assembler_cmd: Option<Arc<AssemblerCommand>> -} - -impl CodegenContext { - pub fn create_diag_handler(&self) -> Handler { - Handler::with_emitter(true, false, Box::new(self.diag_emitter.clone())) - } - - pub(crate) fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Metadata => &self.metadata_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } +pub(crate) fn save_temp_bitcode( + cgcx: &CodegenContext<LlvmCodegenBackend>, + module: &ModuleCodegen<ModuleLlvm>, + name: &str +) { + if !cgcx.save_temps { + return } - - pub(crate) fn save_temp_bitcode(&self, module: &ModuleCodegen<ModuleLlvm>, name: &str) { - if !self.save_temps { - return - } - unsafe { - let ext = format!("{}.bc", name); - let cgu = Some(&module.name[..]); - let path = self.output_filenames.temp_path_ext(&ext, cgu); - let cstr = path2cstr(&path); - let llmod = module.module_llvm.llmod(); - llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); - } + unsafe { + let ext = format!("{}.bc", name); + let cgu = Some(&module.name[..]); + let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); + let cstr = path2cstr(&path); + let llmod = module.module_llvm.llmod(); + llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); } } pub struct DiagnosticHandlers<'a> { - data: *mut (&'a CodegenContext, &'a Handler), + data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a Handler), llcx: &'a llvm::Context, } impl<'a> DiagnosticHandlers<'a> { - pub fn new(cgcx: &'a CodegenContext, + pub fn new(cgcx: &'a CodegenContext<LlvmCodegenBackend>, handler: &'a Handler, llcx: &'a llvm::Context) -> Self { let data = Box::into_raw(Box::new((cgcx, handler))); @@ -451,7 +246,7 @@ impl<'a> Drop for DiagnosticHandlers<'a> { } } -unsafe extern "C" fn report_inline_asm<'a, 'b>(cgcx: &'a CodegenContext, +unsafe extern "C" fn report_inline_asm<'a, 'b>(cgcx: &'a CodegenContext<LlvmCodegenBackend>, msg: &'b str, cookie: c_uint) { cgcx.diag_emitter.inline_asm_error(cookie as u32, msg.to_owned()); @@ -463,7 +258,7 @@ unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, if user.is_null() { return } - let (cgcx, _) = *(user as *const (&CodegenContext, &Handler)); + let (cgcx, _) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); let msg = llvm::build_string(|s| llvm::LLVMRustWriteSMDiagnosticToString(diag, s)) .expect("non-UTF8 SMDiagnostic"); @@ -475,7 +270,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void if user.is_null() { return } - let (cgcx, diag_handler) = *(user as *const (&CodegenContext, &Handler)); + let (cgcx, diag_handler) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler)); match llvm::diagnostic::Diagnostic::unpack(info) { llvm::diagnostic::InlineAsm(inline) => { @@ -512,7 +307,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void } // Unsafe due to LLVM calls. -unsafe fn optimize(cgcx: &CodegenContext, +pub(crate) unsafe fn optimize(cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, module: &ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, @@ -572,7 +367,8 @@ unsafe fn optimize(cgcx: &CodegenContext, if !config.no_prepopulate_passes { llvm::LLVMRustAddAnalysisPasses(tm, fpm, llmod); llvm::LLVMRustAddAnalysisPasses(tm, mpm, llmod); - let opt_level = config.opt_level.unwrap_or(llvm::CodeGenOptLevel::None); + let opt_level = config.opt_level.map(get_llvm_opt_level) + .unwrap_or(llvm::CodeGenOptLevel::None); let prepare_for_thin_lto = cgcx.lto == Lto::Thin || cgcx.lto == Lto::ThinLocal || (cgcx.lto != Lto::Fat && cgcx.opts.debugging_opts.cross_lang_lto.enabled()); have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; @@ -644,35 +440,7 @@ unsafe fn optimize(cgcx: &CodegenContext, Ok(()) } -fn generate_lto_work(cgcx: &CodegenContext, - modules: Vec<ModuleCodegen<ModuleLlvm>>, - import_only_modules: Vec<(SerializedModule, WorkProduct)>) - -> Vec<(WorkItem, u64)> -{ - let mut timeline = cgcx.time_graph.as_ref().map(|tg| { - tg.start(CODEGEN_WORKER_TIMELINE, - CODEGEN_WORK_PACKAGE_KIND, - "generate lto") - }).unwrap_or(Timeline::noop()); - let (lto_modules, copy_jobs) = lto::run(cgcx, modules, import_only_modules, &mut timeline) - .unwrap_or_else(|e| e.raise()); - - let lto_modules = lto_modules.into_iter().map(|module| { - let cost = module.cost(); - (WorkItem::LTO(module), cost) - }); - - let copy_jobs = copy_jobs.into_iter().map(|wp| { - (WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { - name: wp.cgu_name.clone(), - source: wp, - }), 0) - }); - - lto_modules.chain(copy_jobs).collect() -} - -unsafe fn codegen(cgcx: &CodegenContext, +pub(crate) unsafe fn codegen(cgcx: &CodegenContext<LlvmCodegenBackend>, diag_handler: &Handler, module: ModuleCodegen<ModuleLlvm>, config: &ModuleConfig, @@ -879,7 +647,7 @@ unsafe fn codegen(cgcx: &CodegenContext, /// /// Basically all of this is us attempting to follow in the footsteps of clang /// on iOS. See #35968 for lots more info. -unsafe fn embed_bitcode(cgcx: &CodegenContext, +unsafe fn embed_bitcode(cgcx: &CodegenContext<LlvmCodegenBackend>, llcx: &llvm::Context, llmod: &llvm::Module, bitcode: Option<&[u8]>) { @@ -919,1279 +687,6 @@ unsafe fn embed_bitcode(cgcx: &CodegenContext, llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); } -pub(crate) struct CompiledModules { - pub modules: Vec<CompiledModule>, - pub metadata_module: CompiledModule, - pub allocator_module: Option<CompiledModule>, -} - -fn need_crate_bitcode_for_rlib(sess: &Session) -> bool { - sess.crate_types.borrow().contains(&config::CrateType::Rlib) && - sess.opts.output_types.contains_key(&OutputType::Exe) -} - -fn need_pre_thin_lto_bitcode_for_incr_comp(sess: &Session) -> bool { - if sess.opts.incremental.is_none() { - return false - } - - match sess.lto() { - Lto::Fat | - Lto::No => false, - Lto::Thin | - Lto::ThinLocal => true, - } -} - -pub fn start_async_codegen(tcx: TyCtxt, - time_graph: Option<TimeGraph>, - metadata: EncodedMetadata, - coordinator_receive: Receiver<Box<dyn Any + Send>>, - total_cgus: usize) - -> OngoingCodegen { - let sess = tcx.sess; - let crate_name = tcx.crate_name(LOCAL_CRATE); - let crate_hash = tcx.crate_hash(LOCAL_CRATE); - let no_builtins = attr::contains_name(&tcx.hir.krate().attrs, "no_builtins"); - let subsystem = attr::first_attr_value_str_by_name(&tcx.hir.krate().attrs, - "windows_subsystem"); - let windows_subsystem = subsystem.map(|subsystem| { - if subsystem != "windows" && subsystem != "console" { - tcx.sess.fatal(&format!("invalid windows subsystem `{}`, only \ - `windows` and `console` are allowed", - subsystem)); - } - subsystem.to_string() - }); - - let linker_info = LinkerInfo::new(tcx); - let crate_info = CrateInfo::new(tcx); - - // Figure out what we actually need to build. - let mut modules_config = ModuleConfig::new(sess.opts.cg.passes.clone()); - let mut metadata_config = ModuleConfig::new(vec![]); - let mut allocator_config = ModuleConfig::new(vec![]); - - if let Some(ref sanitizer) = sess.opts.debugging_opts.sanitizer { - match *sanitizer { - Sanitizer::Address => { - modules_config.passes.push("asan".to_owned()); - modules_config.passes.push("asan-module".to_owned()); - } - Sanitizer::Memory => { - modules_config.passes.push("msan".to_owned()) - } - Sanitizer::Thread => { - modules_config.passes.push("tsan".to_owned()) - } - _ => {} - } - } - - if sess.opts.debugging_opts.profile { - modules_config.passes.push("insert-gcov-profiling".to_owned()) - } - - modules_config.pgo_gen = sess.opts.debugging_opts.pgo_gen.clone(); - modules_config.pgo_use = sess.opts.debugging_opts.pgo_use.clone(); - - modules_config.opt_level = Some(get_llvm_opt_level(sess.opts.optimize)); - modules_config.opt_size = Some(get_llvm_opt_size(sess.opts.optimize)); - - // Save all versions of the bytecode if we're saving our temporaries. - if sess.opts.cg.save_temps { - modules_config.emit_no_opt_bc = true; - modules_config.emit_pre_thin_lto_bc = true; - modules_config.emit_bc = true; - modules_config.emit_lto_bc = true; - metadata_config.emit_bc = true; - allocator_config.emit_bc = true; - } - - // Emit compressed bitcode files for the crate if we're emitting an rlib. - // Whenever an rlib is created, the bitcode is inserted into the archive in - // order to allow LTO against it. - if need_crate_bitcode_for_rlib(sess) { - modules_config.emit_bc_compressed = true; - allocator_config.emit_bc_compressed = true; - } - - modules_config.emit_pre_thin_lto_bc = - need_pre_thin_lto_bitcode_for_incr_comp(sess); - - modules_config.no_integrated_as = tcx.sess.opts.cg.no_integrated_as || - tcx.sess.target.target.options.no_integrated_as; - - for output_type in sess.opts.output_types.keys() { - match *output_type { - OutputType::Bitcode => { modules_config.emit_bc = true; } - OutputType::LlvmAssembly => { modules_config.emit_ir = true; } - OutputType::Assembly => { - modules_config.emit_asm = true; - // If we're not using the LLVM assembler, this function - // could be invoked specially with output_type_assembly, so - // in this case we still want the metadata object file. - if !sess.opts.output_types.contains_key(&OutputType::Assembly) { - metadata_config.emit_obj = true; - allocator_config.emit_obj = true; - } - } - OutputType::Object => { modules_config.emit_obj = true; } - OutputType::Metadata => { metadata_config.emit_obj = true; } - OutputType::Exe => { - modules_config.emit_obj = true; - metadata_config.emit_obj = true; - allocator_config.emit_obj = true; - }, - OutputType::Mir => {} - OutputType::DepInfo => {} - } - } - - modules_config.set_flags(sess, no_builtins); - metadata_config.set_flags(sess, no_builtins); - allocator_config.set_flags(sess, no_builtins); - - // Exclude metadata and allocator modules from time_passes output, since - // they throw off the "LLVM passes" measurement. - metadata_config.time_passes = false; - allocator_config.time_passes = false; - - let (shared_emitter, shared_emitter_main) = SharedEmitter::new(); - let (codegen_worker_send, codegen_worker_receive) = channel(); - - let coordinator_thread = start_executing_work(tcx, - &crate_info, - shared_emitter, - codegen_worker_send, - coordinator_receive, - total_cgus, - sess.jobserver.clone(), - time_graph.clone(), - Arc::new(modules_config), - Arc::new(metadata_config), - Arc::new(allocator_config)); - - OngoingCodegen { - crate_name, - crate_hash, - metadata, - windows_subsystem, - linker_info, - crate_info, - - time_graph, - coordinator_send: tcx.tx_to_llvm_workers.lock().clone(), - codegen_worker_receive, - shared_emitter_main, - future: coordinator_thread, - output_filenames: tcx.output_filenames(LOCAL_CRATE), - } -} - -fn copy_all_cgu_workproducts_to_incr_comp_cache_dir( - sess: &Session, - compiled_modules: &CompiledModules, -) -> FxHashMap<WorkProductId, WorkProduct> { - let mut work_products = FxHashMap::default(); - - if sess.opts.incremental.is_none() { - return work_products; - } - - for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) { - let mut files = vec![]; - - if let Some(ref path) = module.object { - files.push((WorkProductFileKind::Object, path.clone())); - } - if let Some(ref path) = module.bytecode { - files.push((WorkProductFileKind::Bytecode, path.clone())); - } - if let Some(ref path) = module.bytecode_compressed { - files.push((WorkProductFileKind::BytecodeCompressed, path.clone())); - } - - if let Some((id, product)) = - copy_cgu_workproducts_to_incr_comp_cache_dir(sess, &module.name, &files) - { - work_products.insert(id, product); - } - } - - work_products -} - -fn produce_final_output_artifacts(sess: &Session, - compiled_modules: &CompiledModules, - crate_output: &OutputFilenames) { - let mut user_wants_bitcode = false; - let mut user_wants_objects = false; - - // Produce final compile outputs. - let copy_gracefully = |from: &Path, to: &Path| { - if let Err(e) = fs::copy(from, to) { - sess.err(&format!("could not copy {:?} to {:?}: {}", from, to, e)); - } - }; - - let copy_if_one_unit = |output_type: OutputType, - keep_numbered: bool| { - if compiled_modules.modules.len() == 1 { - // 1) Only one codegen unit. In this case it's no difficulty - // to copy `foo.0.x` to `foo.x`. - let module_name = Some(&compiled_modules.modules[0].name[..]); - let path = crate_output.temp_path(output_type, module_name); - copy_gracefully(&path, - &crate_output.path(output_type)); - if !sess.opts.cg.save_temps && !keep_numbered { - // The user just wants `foo.x`, not `foo.#module-name#.x`. - remove(sess, &path); - } - } else { - let ext = crate_output.temp_path(output_type, None) - .extension() - .unwrap() - .to_str() - .unwrap() - .to_owned(); - - if crate_output.outputs.contains_key(&output_type) { - // 2) Multiple codegen units, with `--emit foo=some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!("ignoring emit path because multiple .{} files \ - were produced", ext)); - } else if crate_output.single_output_file.is_some() { - // 3) Multiple codegen units, with `-o some_name`. We have - // no good solution for this case, so warn the user. - sess.warn(&format!("ignoring -o because multiple .{} files \ - were produced", ext)); - } else { - // 4) Multiple codegen units, but no explicit name. We - // just leave the `foo.0.x` files in place. - // (We don't have to do any work in this case.) - } - } - }; - - // Flag to indicate whether the user explicitly requested bitcode. - // Otherwise, we produced it only as a temporary output, and will need - // to get rid of it. - for output_type in crate_output.outputs.keys() { - match *output_type { - OutputType::Bitcode => { - user_wants_bitcode = true; - // Copy to .bc, but always keep the .0.bc. There is a later - // check to figure out if we should delete .0.bc files, or keep - // them for making an rlib. - copy_if_one_unit(OutputType::Bitcode, true); - } - OutputType::LlvmAssembly => { - copy_if_one_unit(OutputType::LlvmAssembly, false); - } - OutputType::Assembly => { - copy_if_one_unit(OutputType::Assembly, false); - } - OutputType::Object => { - user_wants_objects = true; - copy_if_one_unit(OutputType::Object, true); - } - OutputType::Mir | - OutputType::Metadata | - OutputType::Exe | - OutputType::DepInfo => {} - } - } - - // Clean up unwanted temporary files. - - // We create the following files by default: - // - #crate#.#module-name#.bc - // - #crate#.#module-name#.o - // - #crate#.crate.metadata.bc - // - #crate#.crate.metadata.o - // - #crate#.o (linked from crate.##.o) - // - #crate#.bc (copied from crate.##.bc) - // We may create additional files if requested by the user (through - // `-C save-temps` or `--emit=` flags). - - if !sess.opts.cg.save_temps { - // Remove the temporary .#module-name#.o objects. If the user didn't - // explicitly request bitcode (with --emit=bc), and the bitcode is not - // needed for building an rlib, then we must remove .#module-name#.bc as - // well. - - // Specific rules for keeping .#module-name#.bc: - // - If the user requested bitcode (`user_wants_bitcode`), and - // codegen_units > 1, then keep it. - // - If the user requested bitcode but codegen_units == 1, then we - // can toss .#module-name#.bc because we copied it to .bc earlier. - // - If we're not building an rlib and the user didn't request - // bitcode, then delete .#module-name#.bc. - // If you change how this works, also update back::link::link_rlib, - // where .#module-name#.bc files are (maybe) deleted after making an - // rlib. - let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe); - - let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units() > 1; - - let keep_numbered_objects = needs_crate_object || - (user_wants_objects && sess.codegen_units() > 1); - - for module in compiled_modules.modules.iter() { - if let Some(ref path) = module.object { - if !keep_numbered_objects { - remove(sess, path); - } - } - - if let Some(ref path) = module.bytecode { - if !keep_numbered_bitcode { - remove(sess, path); - } - } - } - - if !user_wants_bitcode { - if let Some(ref path) = compiled_modules.metadata_module.bytecode { - remove(sess, &path); - } - - if let Some(ref allocator_module) = compiled_modules.allocator_module { - if let Some(ref path) = allocator_module.bytecode { - remove(sess, path); - } - } - } - } - - // We leave the following files around by default: - // - #crate#.o - // - #crate#.crate.metadata.o - // - #crate#.bc - // These are used in linking steps and will be cleaned up afterward. -} - -pub(crate) fn dump_incremental_data(_codegen_results: &CodegenResults) { - // FIXME(mw): This does not work at the moment because the situation has - // become more complicated due to incremental LTO. Now a CGU - // can have more than two caching states. - // println!("[incremental] Re-using {} out of {} modules", - // codegen_results.modules.iter().filter(|m| m.pre_existing).count(), - // codegen_results.modules.len()); -} - -enum WorkItem { - /// Optimize a newly codegened, totally unoptimized module. - Optimize(ModuleCodegen<ModuleLlvm>), - /// Copy the post-LTO artifacts from the incremental cache to the output - /// directory. - CopyPostLtoArtifacts(CachedModuleCodegen), - /// Perform (Thin)LTO on the given module. - LTO(lto::LtoModuleCodegen), -} - -impl WorkItem { - fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | - WorkItem::LTO(_) => ModuleKind::Regular, - } - } - - fn name(&self) -> String { - match *self { - WorkItem::Optimize(ref m) => format!("optimize: {}", m.name), - WorkItem::CopyPostLtoArtifacts(ref m) => format!("copy post LTO artifacts: {}", m.name), - WorkItem::LTO(ref m) => format!("lto: {}", m.name()), - } - } -} - -enum WorkItemResult { - Compiled(CompiledModule), - NeedsLTO(ModuleCodegen<ModuleLlvm>), -} - -fn execute_work_item(cgcx: &CodegenContext, - work_item: WorkItem, - timeline: &mut Timeline) - -> Result<WorkItemResult, FatalError> -{ - let module_config = cgcx.config(work_item.module_kind()); - - match work_item { - WorkItem::Optimize(module) => { - execute_optimize_work_item(cgcx, module, module_config, timeline) - } - WorkItem::CopyPostLtoArtifacts(module) => { - execute_copy_from_cache_work_item(cgcx, module, module_config, timeline) - } - WorkItem::LTO(module) => { - execute_lto_work_item(cgcx, module, module_config, timeline) - } - } -} - -fn execute_optimize_work_item(cgcx: &CodegenContext, - module: ModuleCodegen<ModuleLlvm>, - module_config: &ModuleConfig, - timeline: &mut Timeline) - -> Result<WorkItemResult, FatalError> -{ - let diag_handler = cgcx.create_diag_handler(); - - unsafe { - optimize(cgcx, &diag_handler, &module, module_config, timeline)?; - } - - let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled(); - - // After we've done the initial round of optimizations we need to - // decide whether to synchronously codegen this module or ship it - // back to the coordinator thread for further LTO processing (which - // has to wait for all the initial modules to be optimized). - // - // Here we dispatch based on the `cgcx.lto` and kind of module we're - // codegenning... - let needs_lto = match cgcx.lto { - Lto::No => false, - - // If the linker does LTO, we don't have to do it. Note that we - // keep doing full LTO, if it is requested, as not to break the - // assumption that the output will be a single module. - Lto::Thin | Lto::ThinLocal if linker_does_lto => false, - - // Here we've got a full crate graph LTO requested. We ignore - // this, however, if the crate type is only an rlib as there's - // no full crate graph to process, that'll happen later. - // - // This use case currently comes up primarily for targets that - // require LTO so the request for LTO is always unconditionally - // passed down to the backend, but we don't actually want to do - // anything about it yet until we've got a final product. - Lto::Fat | Lto::Thin => { - cgcx.crate_types.len() != 1 || - cgcx.crate_types[0] != config::CrateType::Rlib - } - - // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if - // it shows up. This is due to various linker shenanigans that - // we'll encounter later. - Lto::ThinLocal => { - module.kind != ModuleKind::Allocator - } - }; - - // Metadata modules never participate in LTO regardless of the lto - // settings. - let needs_lto = needs_lto && module.kind != ModuleKind::Metadata; - - if needs_lto { - Ok(WorkItemResult::NeedsLTO(module)) - } else { - let module = unsafe { - codegen(cgcx, &diag_handler, module, module_config, timeline)? - }; - Ok(WorkItemResult::Compiled(module)) - } -} - -fn execute_copy_from_cache_work_item(cgcx: &CodegenContext, - module: CachedModuleCodegen, - module_config: &ModuleConfig, - _: &mut Timeline) - -> Result<WorkItemResult, FatalError> -{ - let incr_comp_session_dir = cgcx.incr_comp_session_dir - .as_ref() - .unwrap(); - let mut object = None; - let mut bytecode = None; - let mut bytecode_compressed = None; - for (kind, saved_file) in &module.source.saved_files { - let obj_out = match kind { - WorkProductFileKind::Object => { - let path = cgcx.output_filenames.temp_path(OutputType::Object, - Some(&module.name)); - object = Some(path.clone()); - path - } - WorkProductFileKind::Bytecode => { - let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, - Some(&module.name)); - bytecode = Some(path.clone()); - path - } - WorkProductFileKind::BytecodeCompressed => { - let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, - Some(&module.name)) - .with_extension(RLIB_BYTECODE_EXTENSION); - bytecode_compressed = Some(path.clone()); - path - } - }; - let source_file = in_incr_comp_dir(&incr_comp_session_dir, - &saved_file); - debug!("copying pre-existing module `{}` from {:?} to {}", - module.name, - source_file, - obj_out.display()); - if let Err(err) = link_or_copy(&source_file, &obj_out) { - let diag_handler = cgcx.create_diag_handler(); - diag_handler.err(&format!("unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), - err)); - } - } - - assert_eq!(object.is_some(), module_config.emit_obj); - assert_eq!(bytecode.is_some(), module_config.emit_bc); - assert_eq!(bytecode_compressed.is_some(), module_config.emit_bc_compressed); - - Ok(WorkItemResult::Compiled(CompiledModule { - name: module.name, - kind: ModuleKind::Regular, - object, - bytecode, - bytecode_compressed, - })) -} - -fn execute_lto_work_item(cgcx: &CodegenContext, - mut module: lto::LtoModuleCodegen, - module_config: &ModuleConfig, - timeline: &mut Timeline) - -> Result<WorkItemResult, FatalError> -{ - let diag_handler = cgcx.create_diag_handler(); - - unsafe { - let module = module.optimize(cgcx, timeline)?; - let module = codegen(cgcx, &diag_handler, module, module_config, timeline)?; - Ok(WorkItemResult::Compiled(module)) - } -} - -enum Message { - Token(io::Result<Acquired>), - NeedsLTO { - result: ModuleCodegen<ModuleLlvm>, - worker_id: usize, - }, - Done { - result: Result<CompiledModule, ()>, - worker_id: usize, - }, - CodegenDone { - llvm_work_item: WorkItem, - cost: u64, - }, - AddImportOnlyModule { - module_data: SerializedModule, - work_product: WorkProduct, - }, - CodegenComplete, - CodegenItem, - CodegenAborted, -} - -struct Diagnostic { - msg: String, - code: Option<DiagnosticId>, - lvl: Level, -} - -#[derive(PartialEq, Clone, Copy, Debug)] -enum MainThreadWorkerState { - Idle, - Codegenning, - LLVMing, -} - -fn start_executing_work(tcx: TyCtxt, - crate_info: &CrateInfo, - shared_emitter: SharedEmitter, - codegen_worker_send: Sender<Message>, - coordinator_receive: Receiver<Box<dyn Any + Send>>, - total_cgus: usize, - jobserver: Client, - time_graph: Option<TimeGraph>, - modules_config: Arc<ModuleConfig>, - metadata_config: Arc<ModuleConfig>, - allocator_config: Arc<ModuleConfig>) - -> thread::JoinHandle<Result<CompiledModules, ()>> { - let coordinator_send = tcx.tx_to_llvm_workers.lock().clone(); - let sess = tcx.sess; - - // Compute the set of symbols we need to retain when doing LTO (if we need to) - let exported_symbols = { - let mut exported_symbols = FxHashMap::default(); - - let copy_symbols = |cnum| { - let symbols = tcx.exported_symbols(cnum) - .iter() - .map(|&(s, lvl)| (s.symbol_name(tcx).to_string(), lvl)) - .collect(); - Arc::new(symbols) - }; - - match sess.lto() { - Lto::No => None, - Lto::ThinLocal => { - exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); - Some(Arc::new(exported_symbols)) - } - Lto::Fat | Lto::Thin => { - exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE)); - for &cnum in tcx.crates().iter() { - exported_symbols.insert(cnum, copy_symbols(cnum)); - } - Some(Arc::new(exported_symbols)) - } - } - }; - - // First up, convert our jobserver into a helper thread so we can use normal - // mpsc channels to manage our messages and such. - // After we've requested tokens then we'll, when we can, - // get tokens on `coordinator_receive` which will - // get managed in the main loop below. - let coordinator_send2 = coordinator_send.clone(); - let helper = jobserver.into_helper_thread(move |token| { - drop(coordinator_send2.send(Box::new(Message::Token(token)))); - }).expect("failed to spawn helper thread"); - - let mut each_linked_rlib_for_lto = Vec::new(); - drop(link::each_linked_rlib(sess, crate_info, &mut |cnum, path| { - if link::ignored_for_lto(sess, crate_info, cnum) { - return - } - each_linked_rlib_for_lto.push((cnum, path.to_path_buf())); - })); - - let assembler_cmd = if modules_config.no_integrated_as { - // HACK: currently we use linker (gcc) as our assembler - let (linker, flavor) = link::linker_and_flavor(sess); - - let (name, mut cmd) = get_linker(sess, &linker, flavor); - cmd.args(&sess.target.target.options.asm_args); - - Some(Arc::new(AssemblerCommand { name, cmd })) - } else { - None - }; - - let cgcx = CodegenContext { - crate_types: sess.crate_types.borrow().clone(), - each_linked_rlib_for_lto, - lto: sess.lto(), - no_landing_pads: sess.no_landing_pads(), - fewer_names: sess.fewer_names(), - save_temps: sess.opts.cg.save_temps, - opts: Arc::new(sess.opts.clone()), - time_passes: sess.time_passes(), - exported_symbols, - plugin_passes: sess.plugin_llvm_passes.borrow().clone(), - remark: sess.opts.cg.remark.clone(), - worker: 0, - incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), - cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(), - coordinator_send, - diag_emitter: shared_emitter.clone(), - time_graph, - output_filenames: tcx.output_filenames(LOCAL_CRATE), - regular_module_config: modules_config, - metadata_module_config: metadata_config, - allocator_module_config: allocator_config, - tm_factory: target_machine_factory(tcx.sess, false), - total_cgus, - msvc_imps_needed: msvc_imps_needed(tcx), - target_pointer_width: tcx.sess.target.target.target_pointer_width.clone(), - debuginfo: tcx.sess.opts.debuginfo, - assembler_cmd, - }; - - // This is the "main loop" of parallel work happening for parallel codegen. - // It's here that we manage parallelism, schedule work, and work with - // messages coming from clients. - // - // There are a few environmental pre-conditions that shape how the system - // is set up: - // - // - Error reporting only can happen on the main thread because that's the - // only place where we have access to the compiler `Session`. - // - LLVM work can be done on any thread. - // - Codegen can only happen on the main thread. - // - Each thread doing substantial work most be in possession of a `Token` - // from the `Jobserver`. - // - The compiler process always holds one `Token`. Any additional `Tokens` - // have to be requested from the `Jobserver`. - // - // Error Reporting - // =============== - // The error reporting restriction is handled separately from the rest: We - // set up a `SharedEmitter` the holds an open channel to the main thread. - // When an error occurs on any thread, the shared emitter will send the - // error message to the receiver main thread (`SharedEmitterMain`). The - // main thread will periodically query this error message queue and emit - // any error messages it has received. It might even abort compilation if - // has received a fatal error. In this case we rely on all other threads - // being torn down automatically with the main thread. - // Since the main thread will often be busy doing codegen work, error - // reporting will be somewhat delayed, since the message queue can only be - // checked in between to work packages. - // - // Work Processing Infrastructure - // ============================== - // The work processing infrastructure knows three major actors: - // - // - the coordinator thread, - // - the main thread, and - // - LLVM worker threads - // - // The coordinator thread is running a message loop. It instructs the main - // thread about what work to do when, and it will spawn off LLVM worker - // threads as open LLVM WorkItems become available. - // - // The job of the main thread is to codegen CGUs into LLVM work package - // (since the main thread is the only thread that can do this). The main - // thread will block until it receives a message from the coordinator, upon - // which it will codegen one CGU, send it to the coordinator and block - // again. This way the coordinator can control what the main thread is - // doing. - // - // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is - // available, it will spawn off a new LLVM worker thread and let it process - // that a WorkItem. When a LLVM worker thread is done with its WorkItem, - // it will just shut down, which also frees all resources associated with - // the given LLVM module, and sends a message to the coordinator that the - // has been completed. - // - // Work Scheduling - // =============== - // The scheduler's goal is to minimize the time it takes to complete all - // work there is, however, we also want to keep memory consumption low - // if possible. These two goals are at odds with each other: If memory - // consumption were not an issue, we could just let the main thread produce - // LLVM WorkItems at full speed, assuring maximal utilization of - // Tokens/LLVM worker threads. However, since codegen usual is faster - // than LLVM processing, the queue of LLVM WorkItems would fill up and each - // WorkItem potentially holds on to a substantial amount of memory. - // - // So the actual goal is to always produce just enough LLVM WorkItems as - // not to starve our LLVM worker threads. That means, once we have enough - // WorkItems in our queue, we can block the main thread, so it does not - // produce more until we need them. - // - // Doing LLVM Work on the Main Thread - // ---------------------------------- - // Since the main thread owns the compiler processes implicit `Token`, it is - // wasteful to keep it blocked without doing any work. Therefore, what we do - // in this case is: We spawn off an additional LLVM worker thread that helps - // reduce the queue. The work it is doing corresponds to the implicit - // `Token`. The coordinator will mark the main thread as being busy with - // LLVM work. (The actual work happens on another OS thread but we just care - // about `Tokens`, not actual threads). - // - // When any LLVM worker thread finishes while the main thread is marked as - // "busy with LLVM work", we can do a little switcheroo: We give the Token - // of the just finished thread to the LLVM worker thread that is working on - // behalf of the main thread's implicit Token, thus freeing up the main - // thread again. The coordinator can then again decide what the main thread - // should do. This allows the coordinator to make decisions at more points - // in time. - // - // Striking a Balance between Throughput and Memory Consumption - // ------------------------------------------------------------ - // Since our two goals, (1) use as many Tokens as possible and (2) keep - // memory consumption as low as possible, are in conflict with each other, - // we have to find a trade off between them. Right now, the goal is to keep - // all workers busy, which means that no worker should find the queue empty - // when it is ready to start. - // How do we do achieve this? Good question :) We actually never know how - // many `Tokens` are potentially available so it's hard to say how much to - // fill up the queue before switching the main thread to LLVM work. Also we - // currently don't have a means to estimate how long a running LLVM worker - // will still be busy with it's current WorkItem. However, we know the - // maximal count of available Tokens that makes sense (=the number of CPU - // cores), so we can take a conservative guess. The heuristic we use here - // is implemented in the `queue_full_enough()` function. - // - // Some Background on Jobservers - // ----------------------------- - // It's worth also touching on the management of parallelism here. We don't - // want to just spawn a thread per work item because while that's optimal - // parallelism it may overload a system with too many threads or violate our - // configuration for the maximum amount of cpu to use for this process. To - // manage this we use the `jobserver` crate. - // - // Job servers are an artifact of GNU make and are used to manage - // parallelism between processes. A jobserver is a glorified IPC semaphore - // basically. Whenever we want to run some work we acquire the semaphore, - // and whenever we're done with that work we release the semaphore. In this - // manner we can ensure that the maximum number of parallel workers is - // capped at any one point in time. - // - // LTO and the coordinator thread - // ------------------------------ - // - // The final job the coordinator thread is responsible for is managing LTO - // and how that works. When LTO is requested what we'll to is collect all - // optimized LLVM modules into a local vector on the coordinator. Once all - // modules have been codegened and optimized we hand this to the `lto` - // module for further optimization. The `lto` module will return back a list - // of more modules to work on, which the coordinator will continue to spawn - // work for. - // - // Each LLVM module is automatically sent back to the coordinator for LTO if - // necessary. There's already optimizations in place to avoid sending work - // back to the coordinator if LTO isn't requested. - return thread::spawn(move || { - // We pretend to be within the top-level LLVM time-passes task here: - set_time_depth(1); - - let max_workers = ::num_cpus::get(); - let mut worker_id_counter = 0; - let mut free_worker_ids = Vec::new(); - let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| { - if let Some(id) = free_worker_ids.pop() { - id - } else { - let id = worker_id_counter; - worker_id_counter += 1; - id - } - }; - - // This is where we collect codegen units that have gone all the way - // through codegen and LLVM. - let mut compiled_modules = vec![]; - let mut compiled_metadata_module = None; - let mut compiled_allocator_module = None; - let mut needs_lto = Vec::new(); - let mut lto_import_only_modules = Vec::new(); - let mut started_lto = false; - let mut codegen_aborted = false; - - // This flag tracks whether all items have gone through codegens - let mut codegen_done = false; - - // This is the queue of LLVM work items that still need processing. - let mut work_items = Vec::<(WorkItem, u64)>::new(); - - // This are the Jobserver Tokens we currently hold. Does not include - // the implicit Token the compiler process owns no matter what. - let mut tokens = Vec::new(); - - let mut main_thread_worker_state = MainThreadWorkerState::Idle; - let mut running = 0; - - let mut llvm_start_time = None; - - // Run the message loop while there's still anything that needs message - // processing. Note that as soon as codegen is aborted we simply want to - // wait for all existing work to finish, so many of the conditions here - // only apply if codegen hasn't been aborted as they represent pending - // work to be done. - while !codegen_done || - running > 0 || - (!codegen_aborted && ( - work_items.len() > 0 || - needs_lto.len() > 0 || - lto_import_only_modules.len() > 0 || - main_thread_worker_state != MainThreadWorkerState::Idle - )) - { - - // While there are still CGUs to be codegened, the coordinator has - // to decide how to utilize the compiler processes implicit Token: - // For codegenning more CGU or for running them through LLVM. - if !codegen_done { - if main_thread_worker_state == MainThreadWorkerState::Idle { - if !queue_full_enough(work_items.len(), running, max_workers) { - // The queue is not full enough, codegen more items: - if let Err(_) = codegen_worker_send.send(Message::CodegenItem) { - panic!("Could not send Message::CodegenItem to main thread") - } - main_thread_worker_state = MainThreadWorkerState::Codegenning; - } else { - // The queue is full enough to not let the worker - // threads starve. Use the implicit Token to do some - // LLVM work too. - let (item, _) = work_items.pop() - .expect("queue empty - queue_full_enough() broken?"); - let cgcx = CodegenContext { - worker: get_worker_id(&mut free_worker_ids), - .. cgcx.clone() - }; - maybe_start_llvm_timer(cgcx.config(item.module_kind()), - &mut llvm_start_time); - main_thread_worker_state = MainThreadWorkerState::LLVMing; - spawn_work(cgcx, item); - } - } - } else if codegen_aborted { - // don't queue up any more work if codegen was aborted, we're - // just waiting for our existing children to finish - } else { - // If we've finished everything related to normal codegen - // then it must be the case that we've got some LTO work to do. - // Perform the serial work here of figuring out what we're - // going to LTO and then push a bunch of work items onto our - // queue to do LTO - if work_items.len() == 0 && - running == 0 && - main_thread_worker_state == MainThreadWorkerState::Idle { - assert!(!started_lto); - assert!(needs_lto.len() + lto_import_only_modules.len() > 0); - started_lto = true; - let modules = mem::replace(&mut needs_lto, Vec::new()); - let import_only_modules = - mem::replace(&mut lto_import_only_modules, Vec::new()); - for (work, cost) in generate_lto_work(&cgcx, modules, import_only_modules) { - let insertion_index = work_items - .binary_search_by_key(&cost, |&(_, cost)| cost) - .unwrap_or_else(|e| e); - work_items.insert(insertion_index, (work, cost)); - if !cgcx.opts.debugging_opts.no_parallel_llvm { - helper.request_token(); - } - } - } - - // In this branch, we know that everything has been codegened, - // so it's just a matter of determining whether the implicit - // Token is free to use for LLVM work. - match main_thread_worker_state { - MainThreadWorkerState::Idle => { - if let Some((item, _)) = work_items.pop() { - let cgcx = CodegenContext { - worker: get_worker_id(&mut free_worker_ids), - .. cgcx.clone() - }; - maybe_start_llvm_timer(cgcx.config(item.module_kind()), - &mut llvm_start_time); - main_thread_worker_state = MainThreadWorkerState::LLVMing; - spawn_work(cgcx, item); - } else { - // There is no unstarted work, so let the main thread - // take over for a running worker. Otherwise the - // implicit token would just go to waste. - // We reduce the `running` counter by one. The - // `tokens.truncate()` below will take care of - // giving the Token back. - debug_assert!(running > 0); - running -= 1; - main_thread_worker_state = MainThreadWorkerState::LLVMing; - } - } - MainThreadWorkerState::Codegenning => { - bug!("codegen worker should not be codegenning after \ - codegen was already completed") - } - MainThreadWorkerState::LLVMing => { - // Already making good use of that token - } - } - } - - // Spin up what work we can, only doing this while we've got available - // parallelism slots and work left to spawn. - while !codegen_aborted && work_items.len() > 0 && running < tokens.len() { - let (item, _) = work_items.pop().unwrap(); - - maybe_start_llvm_timer(cgcx.config(item.module_kind()), - &mut llvm_start_time); - - let cgcx = CodegenContext { - worker: get_worker_id(&mut free_worker_ids), - .. cgcx.clone() - }; - - spawn_work(cgcx, item); - running += 1; - } - - // Relinquish accidentally acquired extra tokens - tokens.truncate(running); - - let msg = coordinator_receive.recv().unwrap(); - match *msg.downcast::<Message>().ok().unwrap() { - // Save the token locally and the next turn of the loop will use - // this to spawn a new unit of work, or it may get dropped - // immediately if we have no more work to spawn. - Message::Token(token) => { - match token { - Ok(token) => { - tokens.push(token); - - if main_thread_worker_state == MainThreadWorkerState::LLVMing { - // If the main thread token is used for LLVM work - // at the moment, we turn that thread into a regular - // LLVM worker thread, so the main thread is free - // to react to codegen demand. - main_thread_worker_state = MainThreadWorkerState::Idle; - running += 1; - } - } - Err(e) => { - let msg = &format!("failed to acquire jobserver token: {}", e); - shared_emitter.fatal(msg); - // Exit the coordinator thread - panic!("{}", msg) - } - } - } - - Message::CodegenDone { llvm_work_item, cost } => { - // We keep the queue sorted by estimated processing cost, - // so that more expensive items are processed earlier. This - // is good for throughput as it gives the main thread more - // time to fill up the queue and it avoids scheduling - // expensive items to the end. - // Note, however, that this is not ideal for memory - // consumption, as LLVM module sizes are not evenly - // distributed. - let insertion_index = - work_items.binary_search_by_key(&cost, |&(_, cost)| cost); - let insertion_index = match insertion_index { - Ok(idx) | Err(idx) => idx - }; - work_items.insert(insertion_index, (llvm_work_item, cost)); - - if !cgcx.opts.debugging_opts.no_parallel_llvm { - helper.request_token(); - } - assert!(!codegen_aborted); - assert_eq!(main_thread_worker_state, - MainThreadWorkerState::Codegenning); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - - Message::CodegenComplete => { - codegen_done = true; - assert!(!codegen_aborted); - assert_eq!(main_thread_worker_state, - MainThreadWorkerState::Codegenning); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - - // If codegen is aborted that means translation was aborted due - // to some normal-ish compiler error. In this situation we want - // to exit as soon as possible, but we want to make sure all - // existing work has finished. Flag codegen as being done, and - // then conditions above will ensure no more work is spawned but - // we'll keep executing this loop until `running` hits 0. - Message::CodegenAborted => { - assert!(!codegen_aborted); - codegen_done = true; - codegen_aborted = true; - assert_eq!(main_thread_worker_state, - MainThreadWorkerState::Codegenning); - } - - // If a thread exits successfully then we drop a token associated - // with that worker and update our `running` count. We may later - // re-acquire a token to continue running more work. We may also not - // actually drop a token here if the worker was running with an - // "ephemeral token" - // - // Note that if the thread failed that means it panicked, so we - // abort immediately. - Message::Done { result: Ok(compiled_module), worker_id } => { - if main_thread_worker_state == MainThreadWorkerState::LLVMing { - main_thread_worker_state = MainThreadWorkerState::Idle; - } else { - running -= 1; - } - - free_worker_ids.push(worker_id); - - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Metadata => { - assert!(compiled_metadata_module.is_none()); - compiled_metadata_module = Some(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - } - } - Message::NeedsLTO { result, worker_id } => { - assert!(!started_lto); - if main_thread_worker_state == MainThreadWorkerState::LLVMing { - main_thread_worker_state = MainThreadWorkerState::Idle; - } else { - running -= 1; - } - free_worker_ids.push(worker_id); - needs_lto.push(result); - } - Message::AddImportOnlyModule { module_data, work_product } => { - assert!(!started_lto); - assert!(!codegen_done); - assert_eq!(main_thread_worker_state, - MainThreadWorkerState::Codegenning); - lto_import_only_modules.push((module_data, work_product)); - main_thread_worker_state = MainThreadWorkerState::Idle; - } - Message::Done { result: Err(()), worker_id: _ } => { - bug!("worker thread panicked"); - } - Message::CodegenItem => { - bug!("the coordinator should not receive codegen requests") - } - } - } - - if let Some(llvm_start_time) = llvm_start_time { - let total_llvm_time = Instant::now().duration_since(llvm_start_time); - // This is the top-level timing for all of LLVM, set the time-depth - // to zero. - set_time_depth(0); - print_time_passes_entry(cgcx.time_passes, - "LLVM passes", - total_llvm_time); - } - - // Regardless of what order these modules completed in, report them to - // the backend in the same order every time to ensure that we're handing - // out deterministic results. - compiled_modules.sort_by(|a, b| a.name.cmp(&b.name)); - - let compiled_metadata_module = compiled_metadata_module - .expect("Metadata module not compiled?"); - - Ok(CompiledModules { - modules: compiled_modules, - metadata_module: compiled_metadata_module, - allocator_module: compiled_allocator_module, - }) - }); - - // A heuristic that determines if we have enough LLVM WorkItems in the - // queue so that the main thread can do LLVM work instead of codegen - fn queue_full_enough(items_in_queue: usize, - workers_running: usize, - max_workers: usize) -> bool { - // Tune me, plz. - items_in_queue > 0 && - items_in_queue >= max_workers.saturating_sub(workers_running / 2) - } - - fn maybe_start_llvm_timer(config: &ModuleConfig, - llvm_start_time: &mut Option<Instant>) { - // We keep track of the -Ztime-passes output manually, - // since the closure-based interface does not fit well here. - if config.time_passes { - if llvm_start_time.is_none() { - *llvm_start_time = Some(Instant::now()); - } - } - } -} - -pub const CODEGEN_WORKER_ID: usize = ::std::usize::MAX; -pub const CODEGEN_WORKER_TIMELINE: time_graph::TimelineId = - time_graph::TimelineId(CODEGEN_WORKER_ID); -pub const CODEGEN_WORK_PACKAGE_KIND: time_graph::WorkPackageKind = - time_graph::WorkPackageKind(&["#DE9597", "#FED1D3", "#FDC5C7", "#B46668", "#88494B"]); -const LLVM_WORK_PACKAGE_KIND: time_graph::WorkPackageKind = - time_graph::WorkPackageKind(&["#7DB67A", "#C6EEC4", "#ACDAAA", "#579354", "#3E6F3C"]); - -fn spawn_work(cgcx: CodegenContext, work: WorkItem) { - let depth = time_depth(); - - thread::spawn(move || { - set_time_depth(depth); - - // Set up a destructor which will fire off a message that we're done as - // we exit. - struct Bomb { - coordinator_send: Sender<Box<dyn Any + Send>>, - result: Option<WorkItemResult>, - worker_id: usize, - } - impl Drop for Bomb { - fn drop(&mut self) { - let worker_id = self.worker_id; - let msg = match self.result.take() { - Some(WorkItemResult::Compiled(m)) => { - Message::Done { result: Ok(m), worker_id } - } - Some(WorkItemResult::NeedsLTO(m)) => { - Message::NeedsLTO { result: m, worker_id } - } - None => Message::Done { result: Err(()), worker_id } - }; - drop(self.coordinator_send.send(Box::new(msg))); - } - } - - let mut bomb = Bomb { - coordinator_send: cgcx.coordinator_send.clone(), - result: None, - worker_id: cgcx.worker, - }; - - // Execute the work itself, and if it finishes successfully then flag - // ourselves as a success as well. - // - // Note that we ignore any `FatalError` coming out of `execute_work_item`, - // as a diagnostic was already sent off to the main thread - just - // surface that there was an error in this worker. - bomb.result = { - let timeline = cgcx.time_graph.as_ref().map(|tg| { - tg.start(time_graph::TimelineId(cgcx.worker), - LLVM_WORK_PACKAGE_KIND, - &work.name()) - }); - let mut timeline = timeline.unwrap_or(Timeline::noop()); - execute_work_item(&cgcx, work, &mut timeline).ok() - }; - }); -} - -pub fn run_assembler(cgcx: &CodegenContext, handler: &Handler, assembly: &Path, object: &Path) { - let assembler = cgcx.assembler_cmd - .as_ref() - .expect("cgcx.assembler_cmd is missing?"); - - let pname = &assembler.name; - let mut cmd = assembler.cmd.clone(); - cmd.arg("-c").arg("-o").arg(object).arg(assembly); - debug!("{:?}", cmd); - - match cmd.output() { - Ok(prog) => { - if !prog.status.success() { - let mut note = prog.stderr.clone(); - note.extend_from_slice(&prog.stdout); - - handler.struct_err(&format!("linking with `{}` failed: {}", - pname.display(), - prog.status)) - .note(&format!("{:?}", &cmd)) - .note(str::from_utf8(¬e[..]).unwrap()) - .emit(); - handler.abort_if_errors(); - } - }, - Err(e) => { - handler.err(&format!("could not exec the linker `{}`: {}", pname.display(), e)); - handler.abort_if_errors(); - } - } -} - pub unsafe fn with_llvm_pmb(llmod: &llvm::Module, config: &ModuleConfig, opt_level: llvm::CodeGenOptLevel, @@ -2203,7 +698,7 @@ pub unsafe fn with_llvm_pmb(llmod: &llvm::Module, // reasonable defaults and prepare it to actually populate the pass // manager. let builder = llvm::LLVMPassManagerBuilderCreate(); - let opt_size = config.opt_size.unwrap_or(llvm::CodeGenOptSizeNone); + let opt_size = config.opt_size.map(get_llvm_opt_size).unwrap_or(llvm::CodeGenOptSizeNone); let inline_threshold = config.inline_threshold; let pgo_gen_path = config.pgo_gen.as_ref().map(|s| { @@ -2271,295 +766,16 @@ pub unsafe fn with_llvm_pmb(llmod: &llvm::Module, llvm::LLVMPassManagerBuilderDispose(builder); } - -enum SharedEmitterMessage { - Diagnostic(Diagnostic), - InlineAsmError(u32, String), - AbortIfErrors, - Fatal(String), -} - -#[derive(Clone)] -pub struct SharedEmitter { - sender: Sender<SharedEmitterMessage>, -} - -pub struct SharedEmitterMain { - receiver: Receiver<SharedEmitterMessage>, -} - -impl SharedEmitter { - pub fn new() -> (SharedEmitter, SharedEmitterMain) { - let (sender, receiver) = channel(); - - (SharedEmitter { sender }, SharedEmitterMain { receiver }) - } - - fn inline_asm_error(&self, cookie: u32, msg: String) { - drop(self.sender.send(SharedEmitterMessage::InlineAsmError(cookie, msg))); - } - - fn fatal(&self, msg: &str) { - drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string()))); - } -} - -impl Emitter for SharedEmitter { - fn emit(&mut self, db: &DiagnosticBuilder) { - drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { - msg: db.message(), - code: db.code.clone(), - lvl: db.level, - }))); - for child in &db.children { - drop(self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic { - msg: child.message(), - code: None, - lvl: child.level, - }))); - } - drop(self.sender.send(SharedEmitterMessage::AbortIfErrors)); - } -} - -impl SharedEmitterMain { - pub fn check(&self, sess: &Session, blocking: bool) { - loop { - let message = if blocking { - match self.receiver.recv() { - Ok(message) => Ok(message), - Err(_) => Err(()), - } - } else { - match self.receiver.try_recv() { - Ok(message) => Ok(message), - Err(_) => Err(()), - } - }; - - match message { - Ok(SharedEmitterMessage::Diagnostic(diag)) => { - let handler = sess.diagnostic(); - match diag.code { - Some(ref code) => { - handler.emit_with_code(&MultiSpan::new(), - &diag.msg, - code.clone(), - diag.lvl); - } - None => { - handler.emit(&MultiSpan::new(), - &diag.msg, - diag.lvl); - } - } - } - Ok(SharedEmitterMessage::InlineAsmError(cookie, msg)) => { - match Mark::from_u32(cookie).expn_info() { - Some(ei) => sess.span_err(ei.call_site, &msg), - None => sess.err(&msg), - } - } - Ok(SharedEmitterMessage::AbortIfErrors) => { - sess.abort_if_errors(); - } - Ok(SharedEmitterMessage::Fatal(msg)) => { - sess.fatal(&msg); - } - Err(_) => { - break; - } - } - - } - } -} - -pub struct OngoingCodegen { - crate_name: Symbol, - crate_hash: Svh, - metadata: EncodedMetadata, - windows_subsystem: Option<String>, - linker_info: LinkerInfo, - crate_info: CrateInfo, - time_graph: Option<TimeGraph>, - coordinator_send: Sender<Box<dyn Any + Send>>, - codegen_worker_receive: Receiver<Message>, - shared_emitter_main: SharedEmitterMain, - future: thread::JoinHandle<Result<CompiledModules, ()>>, - output_filenames: Arc<OutputFilenames>, -} - -impl OngoingCodegen { - pub(crate) fn join( - self, - sess: &Session - ) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) { - self.shared_emitter_main.check(sess, true); - let compiled_modules = match self.future.join() { - Ok(Ok(compiled_modules)) => compiled_modules, - Ok(Err(())) => { - sess.abort_if_errors(); - panic!("expected abort due to worker thread errors") - }, - Err(_) => { - bug!("panic during codegen/LLVM phase"); - } - }; - - sess.cgu_reuse_tracker.check_expected_reuse(sess); - - sess.abort_if_errors(); - - if let Some(time_graph) = self.time_graph { - time_graph.dump(&format!("{}-timings", self.crate_name)); - } - - let work_products = - copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, - &compiled_modules); - produce_final_output_artifacts(sess, - &compiled_modules, - &self.output_filenames); - - // FIXME: time_llvm_passes support - does this use a global context or - // something? - if sess.codegen_units() == 1 && sess.time_llvm_passes() { - unsafe { llvm::LLVMRustPrintPassTimings(); } - } - - (CodegenResults { - crate_name: self.crate_name, - crate_hash: self.crate_hash, - metadata: self.metadata, - windows_subsystem: self.windows_subsystem, - linker_info: self.linker_info, - crate_info: self.crate_info, - - modules: compiled_modules.modules, - allocator_module: compiled_modules.allocator_module, - metadata_module: compiled_modules.metadata_module, - }, work_products) - } - - pub(crate) fn submit_pre_codegened_module_to_llvm(&self, - tcx: TyCtxt, - module: ModuleCodegen<ModuleLlvm>) { - self.wait_for_signal_to_codegen_item(); - self.check_for_errors(tcx.sess); - - // These are generally cheap and won't through off scheduling. - let cost = 0; - submit_codegened_module_to_llvm(tcx, module, cost); - } - - pub fn codegen_finished(&self, tcx: TyCtxt) { - self.wait_for_signal_to_codegen_item(); - self.check_for_errors(tcx.sess); - drop(self.coordinator_send.send(Box::new(Message::CodegenComplete))); - } - - /// Consume this context indicating that codegen was entirely aborted, and - /// we need to exit as quickly as possible. - /// - /// This method blocks the current thread until all worker threads have - /// finished, and all worker threads should have exited or be real close to - /// exiting at this point. - pub fn codegen_aborted(self) { - // Signal to the coordinator it should spawn no more work and start - // shutdown. - drop(self.coordinator_send.send(Box::new(Message::CodegenAborted))); - drop(self.future.join()); - } - - pub fn check_for_errors(&self, sess: &Session) { - self.shared_emitter_main.check(sess, false); - } - - pub fn wait_for_signal_to_codegen_item(&self) { - match self.codegen_worker_receive.recv() { - Ok(Message::CodegenItem) => { - // Nothing to do - } - Ok(_) => panic!("unexpected message"), - Err(_) => { - // One of the LLVM threads must have panicked, fall through so - // error handling can be reached. - } - } - } -} - -// impl Drop for OngoingCodegen { -// fn drop(&mut self) { -// } -// } - -pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt, - module: ModuleCodegen<ModuleLlvm>, - cost: u64) { - let llvm_work_item = WorkItem::Optimize(module); - drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { - llvm_work_item, - cost, - }))); -} - -pub(crate) fn submit_post_lto_module_to_llvm(tcx: TyCtxt, - module: CachedModuleCodegen) { - let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module); - drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone { - llvm_work_item, - cost: 0, - }))); -} - -pub(crate) fn submit_pre_lto_module_to_llvm(tcx: TyCtxt, - module: CachedModuleCodegen) { - let filename = pre_lto_bitcode_filename(&module.name); - let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename); - let file = fs::File::open(&bc_path).unwrap_or_else(|e| { - panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e) - }); - - let mmap = unsafe { - memmap::Mmap::map(&file).unwrap_or_else(|e| { - panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e) - }) - }; - - // Schedule the module to be loaded - drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::AddImportOnlyModule { - module_data: SerializedModule::FromUncompressedFile(mmap), - work_product: module.source, - }))); -} - -pub(super) fn pre_lto_bitcode_filename(module_name: &str) -> String { - format!("{}.{}", module_name, PRE_THIN_LTO_BC_EXT) -} - -fn msvc_imps_needed(tcx: TyCtxt) -> bool { - // This should never be true (because it's not supported). If it is true, - // something is wrong with commandline arg validation. - assert!(!(tcx.sess.opts.debugging_opts.cross_lang_lto.enabled() && - tcx.sess.target.target.options.is_like_msvc && - tcx.sess.opts.cg.prefer_dynamic)); - - tcx.sess.target.target.options.is_like_msvc && - tcx.sess.crate_types.borrow().iter().any(|ct| *ct == config::CrateType::Rlib) && - // ThinLTO can't handle this workaround in all cases, so we don't - // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing - // dynamic linking when cross-language LTO is enabled. - !tcx.sess.opts.debugging_opts.cross_lang_lto.enabled() -} - // Create a `__imp_<symbol> = &symbol` global for every public static `symbol`. // This is required to satisfy `dllimport` references to static data in .rlibs // when using MSVC linker. We do this only for data, as linker can fix up // code references on its own. // See #26591, #27438 -fn create_msvc_imps(cgcx: &CodegenContext, llcx: &llvm::Context, llmod: &llvm::Module) { +fn create_msvc_imps( + cgcx: &CodegenContext<LlvmCodegenBackend>, + llcx: &llvm::Context, + llmod: &llvm::Module +) { if !cgcx.msvc_imps_needed { return } diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 5f8601cb6da..529639bf033 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -28,7 +28,6 @@ use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use super::LlvmCodegenBackend; -use back::write; use llvm; use metadata; use rustc::mir::mono::{Linkage, Visibility, Stats}; @@ -44,6 +43,7 @@ use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_data_structures::small_c_str::SmallCStr; use rustc_codegen_ssa::interfaces::*; +use rustc_codegen_ssa::back::write::submit_codegened_module_to_llvm; use std::ffi::CString; use std::time::Instant; @@ -53,7 +53,7 @@ use rustc::hir::CodegenFnAttrs; use value::Value; -pub(crate) fn write_metadata<'a, 'gcx>( +pub fn write_metadata<'a, 'gcx>( tcx: TyCtxt<'a, 'gcx, 'gcx>, llvm_module: &ModuleLlvm ) -> EncodedMetadata { @@ -163,9 +163,7 @@ pub fn compile_codegen_unit<'ll, 'tcx>(tcx: TyCtxt<'ll, 'tcx, 'tcx>, let cost = time_to_codegen.as_secs() * 1_000_000_000 + time_to_codegen.subsec_nanos() as u64; - write::submit_codegened_module_to_llvm(tcx, - module, - cost); + submit_codegened_module_to_llvm(&LlvmCodegenBackend(()), tcx, module, cost); return stats; fn module_codegen<'ll, 'tcx>( diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index 85a9e551abb..8f14637d6f1 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -68,15 +68,17 @@ extern crate tempfile; extern crate memmap; use rustc_codegen_ssa::interfaces::*; -use time_graph::TimeGraph; -use std::sync::mpsc::Receiver; -use back::write::{self, OngoingCodegen}; +use rustc_codegen_ssa::back::write::{CodegenContext, ModuleConfig}; +use rustc_codegen_ssa::back::lto::{SerializedModule, LtoModuleCodegen, ThinModule}; +use rustc_codegen_ssa::CompiledModule; +use errors::{FatalError, Handler}; +use rustc::dep_graph::WorkProduct; +use rustc::util::time_graph::Timeline; use syntax_pos::symbol::InternedString; use rustc::mir::mono::Stats; - pub use llvm_util::target_features; use std::any::Any; -use std::sync::mpsc; +use std::sync::{mpsc, Arc}; use rustc::dep_graph::DepGraph; use rustc::middle::allocator::AllocatorKind; @@ -87,9 +89,8 @@ use rustc::ty::{self, TyCtxt}; use rustc::util::time_graph; use rustc::util::profiling::ProfileCategory; use rustc_mir::monomorphize; -use rustc_codegen_ssa::{ModuleCodegen, CompiledModule, CachedModuleCodegen, CrateInfo}; +use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_utils::codegen_backend::CodegenBackend; -use rustc_data_structures::svh::Svh; mod diagnostics; @@ -127,12 +128,10 @@ mod type_; mod type_of; mod value; +#[derive(Clone)] pub struct LlvmCodegenBackend(()); impl ExtraBackendMethods for LlvmCodegenBackend { - type Module = ModuleLlvm; - type OngoingCodegen = OngoingCodegen; - fn new_metadata(&self, sess: &Session, mod_name: &str) -> ModuleLlvm { ModuleLlvm::new(sess, mod_name) } @@ -143,57 +142,95 @@ impl ExtraBackendMethods for LlvmCodegenBackend { ) -> EncodedMetadata { base::write_metadata(tcx, metadata) } - fn start_async_codegen( + fn codegen_allocator(&self, tcx: TyCtxt, mods: &ModuleLlvm, kind: AllocatorKind) { + unsafe { allocator::codegen(tcx, mods, kind) } + } + fn compile_codegen_unit<'a, 'tcx: 'a>( &self, - tcx: TyCtxt, - time_graph: Option<TimeGraph>, - metadata: EncodedMetadata, - coordinator_receive: Receiver<Box<dyn Any + Send>>, - total_cgus: usize - ) -> OngoingCodegen { - write::start_async_codegen(tcx, time_graph, metadata, coordinator_receive, total_cgus) + tcx: TyCtxt<'a, 'tcx, 'tcx>, + cgu_name: InternedString, + ) -> Stats { + base::compile_codegen_unit(tcx, cgu_name) } - fn submit_pre_codegened_module_to_backend( + fn target_machine_factory( &self, - codegen: &OngoingCodegen, - tcx: TyCtxt, - module: ModuleCodegen<ModuleLlvm> - ) { - codegen.submit_pre_codegened_module_to_llvm(tcx, module) + sess: &Session, + find_features: bool + ) -> Arc<dyn Fn() -> + Result<&'static mut llvm::TargetMachine, String> + Send + Sync> { + back::write::target_machine_factory(sess, find_features) } - fn submit_pre_lto_module_to_backend(&self, tcx: TyCtxt, module: CachedModuleCodegen) { - write::submit_pre_lto_module_to_llvm(tcx, module) + fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { + llvm_util::target_cpu(sess) } - fn submit_post_lto_module_to_backend(&self, tcx: TyCtxt, module: CachedModuleCodegen) { - write::submit_post_lto_module_to_llvm(tcx, module) +} + +impl Clone for &'static mut llvm::TargetMachine { + fn clone(&self) -> Self { + // This method should never be called. It is put here because in + // rustc_codegen_ssa::back::write::CodegenContext, the TargetMachine is contained in a + // closure returned by a function under an Arc. The clone-deriving algorithm works when the + // struct contains the original LLVM TargetMachine type but not any more when supplied with + // a generic type. Hence this dummy Clone implementation. + panic!() } - fn codegen_aborted(codegen: OngoingCodegen) { - codegen.codegen_aborted(); +} + +impl WriteBackendMethods for LlvmCodegenBackend { + type Module = ModuleLlvm; + type ModuleBuffer = back::lto::ModuleBuffer; + type Context = llvm::Context; + type TargetMachine = &'static mut llvm::TargetMachine; + type ThinData = back::lto::ThinData; + type ThinBuffer = back::lto::ThinBuffer; + fn print_pass_timings(&self) { + unsafe { llvm::LLVMRustPrintPassTimings(); } } - fn codegen_finished(&self, codegen: &OngoingCodegen, tcx: TyCtxt) { - codegen.codegen_finished(tcx) + fn run_lto( + cgcx: &CodegenContext<Self>, + modules: Vec<ModuleCodegen<Self::Module>>, + cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>, + timeline: &mut Timeline + ) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError> { + back::lto::run(cgcx, modules, cached_modules, timeline) } - fn check_for_errors(&self, codegen: &OngoingCodegen, sess: &Session) { - codegen.check_for_errors(sess) + unsafe fn optimize( + cgcx: &CodegenContext<Self>, + diag_handler: &Handler, + module: &ModuleCodegen<Self::Module>, + config: &ModuleConfig, + timeline: &mut Timeline + ) -> Result<(), FatalError> { + back::write::optimize(cgcx, diag_handler, module, config, timeline) } - fn codegen_allocator(&self, tcx: TyCtxt, mods: &ModuleLlvm, kind: AllocatorKind) { - unsafe { allocator::codegen(tcx, mods, kind) } + unsafe fn optimize_thin( + cgcx: &CodegenContext<Self>, + thin: &mut ThinModule<Self>, + timeline: &mut Timeline + ) -> Result<ModuleCodegen<Self::Module>, FatalError> { + back::lto::optimize_thin_module(thin, cgcx, timeline) } - fn wait_for_signal_to_codegen_item(&self, codegen: &OngoingCodegen) { - codegen.wait_for_signal_to_codegen_item() + unsafe fn codegen( + cgcx: &CodegenContext<Self>, + diag_handler: &Handler, + module: ModuleCodegen<Self::Module>, + config: &ModuleConfig, + timeline: &mut Timeline + ) -> Result<CompiledModule, FatalError> { + back::write::codegen(cgcx, diag_handler, module, config, timeline) } - fn compile_codegen_unit<'a, 'tcx: 'a>( - &self, - tcx: TyCtxt<'a, 'tcx, 'tcx>, - cgu_name: InternedString, - ) -> Stats { - base::compile_codegen_unit(tcx, cgu_name) + fn run_lto_pass_manager( + cgcx: &CodegenContext<Self>, + module: &ModuleCodegen<Self::Module>, + config: &ModuleConfig, + thin: bool + ) { + back::lto::run_pass_manager(cgcx, module, config, thin) } } - -impl !Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis -impl !Sync for LlvmCodegenBackend {} +unsafe impl<'a> Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis +unsafe impl<'a> Sync for LlvmCodegenBackend {} impl LlvmCodegenBackend { pub fn new() -> Box<dyn CodegenBackend> { @@ -201,7 +238,7 @@ impl LlvmCodegenBackend { } } -impl CodegenBackend for LlvmCodegenBackend { +impl<'a> CodegenBackend for LlvmCodegenBackend { fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -254,21 +291,21 @@ impl CodegenBackend for LlvmCodegenBackend { } fn provide(&self, providers: &mut ty::query::Providers) { - rustc_codegen_utils::symbol_export::provide(providers); rustc_codegen_utils::symbol_names::provide(providers); + rustc_codegen_ssa::back::symbol_export::provide(providers); rustc_codegen_ssa::base::provide_both(providers); attributes::provide(providers); } fn provide_extern(&self, providers: &mut ty::query::Providers) { - rustc_codegen_utils::symbol_export::provide_extern(providers); + rustc_codegen_ssa::back::symbol_export::provide_extern(providers); rustc_codegen_ssa::base::provide_both(providers); attributes::provide_extern(providers); } - fn codegen_crate<'a, 'tcx>( + fn codegen_crate<'b, 'tcx>( &self, - tcx: TyCtxt<'a, 'tcx, 'tcx>, + tcx: TyCtxt<'b, 'tcx, 'tcx>, rx: mpsc::Receiver<Box<dyn Any + Send>> ) -> Box<dyn Any> { box rustc_codegen_ssa::base::codegen_crate(LlvmCodegenBackend(()), tcx, rx) @@ -282,12 +319,13 @@ impl CodegenBackend for LlvmCodegenBackend { outputs: &OutputFilenames, ) -> Result<(), CompileIncomplete>{ use rustc::util::common::time; - let (ongoing_codegen, work_products) = - ongoing_codegen.downcast::<::back::write::OngoingCodegen>() + let (codegen_results, work_products) = + ongoing_codegen.downcast:: + <rustc_codegen_ssa::back::write::OngoingCodegen<LlvmCodegenBackend>>() .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box<Any>") .join(sess); if sess.opts.debugging_opts.incremental_info { - back::write::dump_incremental_data(&ongoing_codegen); + rustc_codegen_ssa::back::write::dump_incremental_data(&codegen_results); } time(sess, @@ -305,14 +343,14 @@ impl CodegenBackend for LlvmCodegenBackend { // This should produce either a finished executable or library. sess.profiler(|p| p.start_activity(ProfileCategory::Linking)); time(sess, "linking", || { - back::link::link_binary(sess, &ongoing_codegen, - outputs, &ongoing_codegen.crate_name.as_str()); + back::link::link_binary(sess, &codegen_results, + outputs, &codegen_results.crate_name.as_str()); }); sess.profiler(|p| p.end_activity(ProfileCategory::Linking)); // Now that we won't touch anything in the incremental compilation directory // any more, we can finalize it (which involves renaming it) - rustc_incremental::finalize_session_directory(sess, ongoing_codegen.crate_hash); + rustc_incremental::finalize_session_directory(sess, codegen_results.crate_hash); Ok(()) } @@ -363,15 +401,4 @@ impl Drop for ModuleLlvm { } } -struct CodegenResults { - crate_name: Symbol, - modules: Vec<CompiledModule>, - allocator_module: Option<CompiledModule>, - metadata_module: CompiledModule, - crate_hash: Svh, - metadata: rustc::middle::cstore::EncodedMetadata, - windows_subsystem: Option<String>, - linker_info: rustc_codegen_utils::linker::LinkerInfo, - crate_info: CrateInfo, -} __build_diagnostic_array! { librustc_codegen_llvm, DIAGNOSTICS } |