diff options
Diffstat (limited to 'src/librustc_codegen_llvm/back/lto.rs')
| -rw-r--r-- | src/librustc_codegen_llvm/back/lto.rs | 178 |
1 files changed, 125 insertions, 53 deletions
diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index d7297ed4176..d3e3441b087 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -1,4 +1,3 @@ -use crate::back::bytecode::DecodedBytecode; use crate::back::write::{ self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, }; @@ -6,19 +5,19 @@ use crate::llvm::archive_ro::ArchiveRO; use crate::llvm::{self, False, True}; use crate::{LlvmCodegenBackend, ModuleLlvm}; use log::{debug, info}; -use rustc::bug; -use rustc::dep_graph::WorkProduct; -use rustc::middle::exported_symbols::SymbolExportLevel; -use rustc::session::config::{self, Lto}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, ModuleConfig}; use rustc_codegen_ssa::traits::*; -use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, RLIB_BYTECODE_EXTENSION}; +use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; +use rustc_middle::dep_graph::WorkProduct; +use rustc_middle::middle::exported_symbols::SymbolExportLevel; use rustc_session::cgu_reuse_tracker::CguReuse; +use rustc_session::config::{self, CrateType, Lto}; use std::ffi::{CStr, CString}; use std::fs::File; @@ -34,13 +33,10 @@ use std::sync::Arc; /// compilation session. pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-imports.bin"; -pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { +pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { - config::CrateType::Executable - | config::CrateType::Staticlib - | config::CrateType::Cdylib => true, - - config::CrateType::Dylib | config::CrateType::Rlib | config::CrateType::ProcMacro => false, + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, + CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, } } @@ -111,22 +107,19 @@ fn prepare_lto( } let archive = ArchiveRO::open(&path).expect("wanted an rlib"); - let bytecodes = archive + let obj_files = archive .iter() .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) - .filter(|&(name, _)| name.ends_with(RLIB_BYTECODE_EXTENSION)); - for (name, data) in bytecodes { - let _timer = - cgcx.prof.generic_activity_with_arg("LLVM_lto_load_upstream_bitcode", name); - info!("adding bytecode {}", name); - let bc_encoded = data.data(); - - let (bc, id) = match DecodedBytecode::new(bc_encoded) { - Ok(b) => Ok((b.bytecode(), b.identifier().to_string())), - Err(e) => Err(diag_handler.fatal(&e)), - }?; - let bc = SerializedModule::FromRlib(bc); - upstream_modules.push((bc, CString::new(id).unwrap())); + .filter(|&(name, _)| looks_like_rust_object_file(name)); + for (name, child) in obj_files { + info!("adding bitcode from {}", name); + match get_bitcode_slice_from_object_data(child.data()) { + Ok(data) => { + let module = SerializedModule::FromRlib(data.to_vec()); + upstream_modules.push((module, CString::new(name).unwrap())); + } + Err(msg) => return Err(diag_handler.fatal(&msg)), + } } } } @@ -134,6 +127,26 @@ fn prepare_lto( Ok((symbol_white_list, upstream_modules)) } +fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { + let mut len = 0; + let data = + unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; + if !data.is_null() { + assert!(len != 0); + let bc = unsafe { slice::from_raw_parts(data, len) }; + + // `bc` must be a sub-slice of `obj`. + assert!(obj.as_ptr() <= bc.as_ptr()); + assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); + + Ok(bc) + } else { + assert!(len == 0); + let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); + Err(format!("failed to get bitcode from object file for LTO ({})", msg)) + } +} + /// Performs fat LTO by merging all modules into a single one and returning it /// for further optimization. pub(crate) fn run_fat( @@ -237,7 +250,7 @@ fn fat_lto( let module: ModuleCodegen<ModuleLlvm> = match costliest_module { Some((_cost, i)) => in_memory.remove(i), None => { - assert!(serialized_modules.len() > 0, "must have at least one serialized module"); + assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); ModuleCodegen { @@ -463,15 +476,18 @@ fn thin_lto( // If previous imports have been deleted, or we get an IO error // reading the file storing them, then we'll just use `None` as the // prev_import_map, which will force the code to be recompiled. - let prev = - if path.exists() { ThinLTOImports::load_from_file(&path).ok() } else { None }; - let curr = ThinLTOImports::from_thin_lto_data(data); + let prev = if path.exists() { + ThinLTOImportMaps::load_from_file(&path).ok() + } else { + None + }; + let curr = ThinLTOImportMaps::from_thin_lto_data(data); (Some(path), prev, curr) } else { // If we don't compile incrementally, we don't need to load the // import data from LLVM. assert!(green_modules.is_empty()); - let curr = ThinLTOImports::default(); + let curr = ThinLTOImportMaps::default(); (None, None, curr) }; info!("thin LTO import map loaded"); @@ -497,31 +513,58 @@ fn thin_lto( let module_name = module_name_to_str(module_name); // If (1.) the module hasn't changed, and (2.) none of the modules - // it imports from has changed, *and* (3.) the import-set itself has - // not changed from the previous compile when it was last - // ThinLTO'ed, then we can re-use the post-ThinLTO version of the - // module. Otherwise, freshly perform LTO optimization. + // it imports from have changed, *and* (3.) the import and export + // sets themselves have not changed from the previous compile when + // it was last ThinLTO'ed, then we can re-use the post-ThinLTO + // version of the module. Otherwise, freshly perform LTO + // optimization. + // + // (Note that globally, the export set is just the inverse of the + // import set.) + // + // For further justification of why the above is necessary and sufficient, + // see the LLVM blog post on ThinLTO: + // + // http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html + // + // which states the following: + // + // ```quote + // any particular ThinLTO backend must be redone iff: + // + // 1. The corresponding (primary) module’s bitcode changed + // 2. The list of imports into or exports from the module changed + // 3. The bitcode for any module being imported from has changed + // 4. Any global analysis result affecting either the primary module + // or anything it imports has changed. + // ``` // // This strategy means we can always save the computed imports as // canon: when we reuse the post-ThinLTO version, condition (3.) - // ensures that the curent import set is the same as the previous + // ensures that the current import set is the same as the previous // one. (And of course, when we don't reuse the post-ThinLTO // version, the current import set *is* the correct one, since we // are doing the ThinLTO in this current compilation cycle.) // - // See rust-lang/rust#59535. + // For more discussion, see rust-lang/rust#59535 (where the import + // issue was discovered) and rust-lang/rust#69798 (where the + // analogous export issue was discovered). if let (Some(prev_import_map), true) = (prev_import_map.as_ref(), green_modules.contains_key(module_name)) { assert!(cgcx.incr_comp_session_dir.is_some()); - let prev_imports = prev_import_map.modules_imported_by(module_name); - let curr_imports = curr_import_map.modules_imported_by(module_name); + let prev_imports = prev_import_map.imports_of(module_name); + let curr_imports = curr_import_map.imports_of(module_name); + let prev_exports = prev_import_map.exports_of(module_name); + let curr_exports = curr_import_map.exports_of(module_name); let imports_all_green = curr_imports .iter() .all(|imported_module| green_modules.contains_key(imported_module)); - - if imports_all_green && equivalent_as_sets(prev_imports, curr_imports) { + if imports_all_green + && equivalent_as_sets(prev_imports, curr_imports) + && equivalent_as_sets(prev_exports, curr_exports) + { let work_product = green_modules[module_name].clone(); copy_jobs.push(work_product); info!(" - {}: re-used", module_name); @@ -538,7 +581,7 @@ fn thin_lto( })); } - // Save the curent ThinLTO import information for the next compilation + // Save the current ThinLTO import information for the next compilation // session, overwriting the previous serialized imports (if any). if let Some(path) = import_map_path { if let Err(err) = curr_import_map.save_to_file(&path) { @@ -881,17 +924,32 @@ pub unsafe fn optimize_thin_module( Ok(module) } +/// Summarizes module import/export relationships used by LLVM's ThinLTO pass. +/// +/// Note that we tend to have two such instances of `ThinLTOImportMaps` in use: +/// one loaded from a file that represents the relationships used during the +/// compilation associated with the incremetnal build artifacts we are +/// attempting to reuse, and another constructed via `from_thin_lto_data`, which +/// captures the relationships of ThinLTO in the current compilation. #[derive(Debug, Default)] -pub struct ThinLTOImports { +pub struct ThinLTOImportMaps { // key = llvm name of importing module, value = list of modules it imports from imports: FxHashMap<String, Vec<String>>, + // key = llvm name of exporting module, value = list of modules it exports to + exports: FxHashMap<String, Vec<String>>, } -impl ThinLTOImports { - fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { +impl ThinLTOImportMaps { + /// Returns modules imported by `llvm_module_name` during some ThinLTO pass. + fn imports_of(&self, llvm_module_name: &str) -> &[String] { self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) } + /// Returns modules exported by `llvm_module_name` during some ThinLTO pass. + fn exports_of(&self, llvm_module_name: &str) -> &[String] { + self.exports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) + } + fn save_to_file(&self, path: &Path) -> io::Result<()> { use std::io::Write; let file = File::create(path)?; @@ -906,18 +964,22 @@ impl ThinLTOImports { Ok(()) } - fn load_from_file(path: &Path) -> io::Result<ThinLTOImports> { + fn load_from_file(path: &Path) -> io::Result<ThinLTOImportMaps> { use std::io::BufRead; let mut imports = FxHashMap::default(); - let mut current_module = None; - let mut current_imports = vec![]; + let mut exports: FxHashMap<_, Vec<_>> = FxHashMap::default(); + let mut current_module: Option<String> = None; + let mut current_imports: Vec<String> = vec![]; let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; if line.is_empty() { let importing_module = current_module.take().expect("Importing module not set"); + for imported in ¤t_imports { + exports.entry(imported.clone()).or_default().push(importing_module.clone()); + } imports.insert(importing_module, mem::replace(&mut current_imports, vec![])); - } else if line.starts_with(" ") { + } else if line.starts_with(' ') { // Space marks an imported module assert_ne!(current_module, None); current_imports.push(line.trim().to_string()); @@ -927,17 +989,17 @@ impl ThinLTOImports { current_module = Some(line.trim().to_string()); } } - Ok(ThinLTOImports { imports }) + Ok(ThinLTOImportMaps { imports, exports }) } /// Loads the ThinLTO import map from ThinLTOData. - unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { + unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImportMaps { unsafe extern "C" fn imported_module_callback( payload: *mut libc::c_void, importing_module_name: *const libc::c_char, imported_module_name: *const libc::c_char, ) { - let map = &mut *(payload as *mut ThinLTOImports); + let map = &mut *(payload as *mut ThinLTOImportMaps); let importing_module_name = CStr::from_ptr(importing_module_name); let importing_module_name = module_name_to_str(&importing_module_name); let imported_module_name = CStr::from_ptr(imported_module_name); @@ -951,8 +1013,18 @@ impl ThinLTOImports { .get_mut(importing_module_name) .unwrap() .push(imported_module_name.to_owned()); + + if !map.exports.contains_key(imported_module_name) { + map.exports.insert(imported_module_name.to_owned(), vec![]); + } + + map.exports + .get_mut(imported_module_name) + .unwrap() + .push(importing_module_name.to_owned()); } - let mut map = ThinLTOImports::default(); + + let mut map = ThinLTOImportMaps::default(); llvm::LLVMRustGetThinLTOModuleImports( data, imported_module_callback, |