diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/attributes.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/back/lto.rs | 261 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/context.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/lib.rs | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/llvm_util.rs | 24 |
6 files changed, 113 insertions, 199 deletions
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index f02c30c3ee3..8d131fa294b 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -194,6 +194,18 @@ pub fn apply_target_cpu_attr(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { ); } +pub fn apply_tune_cpu_attr(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) { + if let Some(tune) = llvm_util::tune_cpu(cx.tcx.sess) { + let tune_cpu = SmallCStr::new(tune); + llvm::AddFunctionAttrStringValue( + llfn, + llvm::AttributePlace::Function, + const_cstr!("tune-cpu"), + tune_cpu.as_c_str(), + ); + } +} + /// Sets the `NonLazyBind` LLVM attribute on a given function, /// assuming the codegen options allow skipping the PLT. pub fn non_lazy_bind(sess: &Session, llfn: &'ll Value) { @@ -303,6 +315,9 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). apply_target_cpu_attr(cx, llfn); + // tune-cpu is only conveyed through the attribute for our purpose. + // The target doesn't care; the subtarget reads our attribute. + apply_tune_cpu_attr(cx, llfn); let features = llvm_target_features(cx.tcx.sess) .map(|s| s.to_string()) diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 4b2d5907a02..ff312bade25 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -2,14 +2,14 @@ use crate::back::write::{ self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, }; use crate::llvm::archive_ro::ArchiveRO; -use crate::llvm::{self, False, True}; +use crate::llvm::{self, build_string, False, True}; use crate::{LlvmCodegenBackend, ModuleLlvm}; use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; use rustc_codegen_ssa::back::symbol_export; use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, ModuleConfig}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashMap; use rustc_errors::{FatalError, Handler}; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; @@ -22,16 +22,14 @@ use tracing::{debug, info}; use std::ffi::{CStr, CString}; use std::fs::File; use std::io; -use std::mem; use std::path::Path; use std::ptr; use std::slice; use std::sync::Arc; -/// We keep track of past LTO imports that were used to produce the current set -/// of compiled object files that we might choose to reuse during this -/// compilation session. -pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-imports.bin"; +/// We keep track of the computed LTO cache keys from the previous +/// session to determine which CGUs we can reuse. +pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { match crate_type { @@ -485,31 +483,31 @@ fn thin_lto( ) .ok_or_else(|| write::llvm_err(&diag_handler, "failed to prepare thin LTO context"))?; - info!("thin LTO data created"); + let data = ThinData(data); - let (import_map_path, prev_import_map, curr_import_map) = - if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { - let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); - // If previous imports have been deleted, or we get an IO error - // reading the file storing them, then we'll just use `None` as the - // prev_import_map, which will force the code to be recompiled. - let prev = if path.exists() { - ThinLTOImportMaps::load_from_file(&path).ok() - } else { - None - }; - let curr = ThinLTOImportMaps::from_thin_lto_data(data); - (Some(path), prev, curr) - } else { - // If we don't compile incrementally, we don't need to load the - // import data from LLVM. - assert!(green_modules.is_empty()); - let curr = ThinLTOImportMaps::default(); - (None, None, curr) - }; - info!("thin LTO import map loaded"); + info!("thin LTO data created"); - let data = ThinData(data); + let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) = + cgcx.incr_comp_session_dir + { + let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); + // If the previous file was deleted, or we get an IO error + // reading the file, then we'll just use `None` as the + // prev_key_map, which will force the code to be recompiled. + let prev = + if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None }; + let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names); + (Some(path), prev, curr) + } else { + // If we don't compile incrementally, we don't need to load the + // import data from LLVM. + assert!(green_modules.is_empty()); + let curr = ThinLTOKeysMap::default(); + (None, None, curr) + }; + info!("thin LTO cache key map loaded"); + info!("prev_key_map: {:#?}", prev_key_map); + info!("curr_key_map: {:#?}", curr_key_map); // Throw our data in an `Arc` as we'll be sharing it across threads. We // also put all memory referenced by the C++ data (buffers, ids, etc) @@ -528,60 +526,14 @@ fn thin_lto( info!("checking which modules can be-reused and which have to be re-optimized."); for (module_index, module_name) in shared.module_names.iter().enumerate() { let module_name = module_name_to_str(module_name); - - // If (1.) the module hasn't changed, and (2.) none of the modules - // it imports from have changed, *and* (3.) the import and export - // sets themselves have not changed from the previous compile when - // it was last ThinLTO'ed, then we can re-use the post-ThinLTO - // version of the module. Otherwise, freshly perform LTO - // optimization. - // - // (Note that globally, the export set is just the inverse of the - // import set.) - // - // For further justification of why the above is necessary and sufficient, - // see the LLVM blog post on ThinLTO: - // - // http://blog.llvm.org/2016/06/thinlto-scalable-and-incremental-lto.html - // - // which states the following: - // - // ```quote - // any particular ThinLTO backend must be redone iff: - // - // 1. The corresponding (primary) module’s bitcode changed - // 2. The list of imports into or exports from the module changed - // 3. The bitcode for any module being imported from has changed - // 4. Any global analysis result affecting either the primary module - // or anything it imports has changed. - // ``` - // - // This strategy means we can always save the computed imports as - // canon: when we reuse the post-ThinLTO version, condition (3.) - // ensures that the current import set is the same as the previous - // one. (And of course, when we don't reuse the post-ThinLTO - // version, the current import set *is* the correct one, since we - // are doing the ThinLTO in this current compilation cycle.) - // - // For more discussion, see rust-lang/rust#59535 (where the import - // issue was discovered) and rust-lang/rust#69798 (where the - // analogous export issue was discovered). - if let (Some(prev_import_map), true) = - (prev_import_map.as_ref(), green_modules.contains_key(module_name)) + if let (Some(prev_key_map), true) = + (prev_key_map.as_ref(), green_modules.contains_key(module_name)) { assert!(cgcx.incr_comp_session_dir.is_some()); - let prev_imports = prev_import_map.imports_of(module_name); - let curr_imports = curr_import_map.imports_of(module_name); - let prev_exports = prev_import_map.exports_of(module_name); - let curr_exports = curr_import_map.exports_of(module_name); - let imports_all_green = curr_imports - .iter() - .all(|imported_module| green_modules.contains_key(imported_module)); - if imports_all_green - && equivalent_as_sets(prev_imports, curr_imports) - && equivalent_as_sets(prev_exports, curr_exports) - { + // If a module exists in both the current and the previous session, + // and has the same LTO cache key in both sessions, then we can re-use it + if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) { let work_product = green_modules[module_name].clone(); copy_jobs.push(work_product); info!(" - {}: re-used", module_name); @@ -599,10 +551,10 @@ fn thin_lto( } // Save the current ThinLTO import information for the next compilation - // session, overwriting the previous serialized imports (if any). - if let Some(path) = import_map_path { - if let Err(err) = curr_import_map.save_to_file(&path) { - let msg = format!("Error while writing ThinLTO import data: {}", err); + // session, overwriting the previous serialized data (if any). + if let Some(path) = key_map_path { + if let Err(err) = curr_key_map.save_to_file(&path) { + let msg = format!("Error while writing ThinLTO key data: {}", err); return Err(write::llvm_err(&diag_handler, &msg)); } } @@ -611,24 +563,6 @@ fn thin_lto( } } -/// Given two slices, each with no repeat elements. returns true if and only if -/// the two slices have the same contents when considered as sets (i.e. when -/// element order is disregarded). -fn equivalent_as_sets(a: &[String], b: &[String]) -> bool { - // cheap path: unequal lengths means cannot possibly be set equivalent. - if a.len() != b.len() { - return false; - } - // fast path: before building new things, check if inputs are equivalent as is. - if a == b { - return true; - } - // slow path: general set comparison. - let a: FxHashSet<&str> = a.iter().map(|s| s.as_str()).collect(); - let b: FxHashSet<&str> = b.iter().map(|s| s.as_str()).collect(); - a == b -} - pub(crate) fn run_pass_manager( cgcx: &CodegenContext<LlvmCodegenBackend>, module: &ModuleCodegen<ModuleLlvm>, @@ -942,113 +876,56 @@ pub unsafe fn optimize_thin_module( Ok(module) } -/// Summarizes module import/export relationships used by LLVM's ThinLTO pass. -/// -/// Note that we tend to have two such instances of `ThinLTOImportMaps` in use: -/// one loaded from a file that represents the relationships used during the -/// compilation associated with the incremetnal build artifacts we are -/// attempting to reuse, and another constructed via `from_thin_lto_data`, which -/// captures the relationships of ThinLTO in the current compilation. +/// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys #[derive(Debug, Default)] -pub struct ThinLTOImportMaps { - // key = llvm name of importing module, value = list of modules it imports from - imports: FxHashMap<String, Vec<String>>, - // key = llvm name of exporting module, value = list of modules it exports to - exports: FxHashMap<String, Vec<String>>, +pub struct ThinLTOKeysMap { + // key = llvm name of importing module, value = LLVM cache key + keys: FxHashMap<String, String>, } -impl ThinLTOImportMaps { - /// Returns modules imported by `llvm_module_name` during some ThinLTO pass. - fn imports_of(&self, llvm_module_name: &str) -> &[String] { - self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) - } - - /// Returns modules exported by `llvm_module_name` during some ThinLTO pass. - fn exports_of(&self, llvm_module_name: &str) -> &[String] { - self.exports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) - } - +impl ThinLTOKeysMap { fn save_to_file(&self, path: &Path) -> io::Result<()> { use std::io::Write; let file = File::create(path)?; let mut writer = io::BufWriter::new(file); - for (importing_module_name, imported_modules) in &self.imports { - writeln!(writer, "{}", importing_module_name)?; - for imported_module in imported_modules { - writeln!(writer, " {}", imported_module)?; - } - writeln!(writer)?; + for (module, key) in &self.keys { + writeln!(writer, "{} {}", module, key)?; } Ok(()) } - fn load_from_file(path: &Path) -> io::Result<ThinLTOImportMaps> { + fn load_from_file(path: &Path) -> io::Result<Self> { use std::io::BufRead; - let mut imports = FxHashMap::default(); - let mut exports: FxHashMap<_, Vec<_>> = FxHashMap::default(); - let mut current_module: Option<String> = None; - let mut current_imports: Vec<String> = vec![]; + let mut keys = FxHashMap::default(); let file = File::open(path)?; for line in io::BufReader::new(file).lines() { let line = line?; - if line.is_empty() { - let importing_module = current_module.take().expect("Importing module not set"); - for imported in ¤t_imports { - exports.entry(imported.clone()).or_default().push(importing_module.clone()); - } - imports.insert(importing_module, mem::replace(&mut current_imports, vec![])); - } else if line.starts_with(' ') { - // Space marks an imported module - assert_ne!(current_module, None); - current_imports.push(line.trim().to_string()); - } else { - // Otherwise, beginning of a new module (must be start or follow empty line) - assert_eq!(current_module, None); - current_module = Some(line.trim().to_string()); - } + let mut split = line.split(" "); + let module = split.next().unwrap(); + let key = split.next().unwrap(); + assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); + keys.insert(module.to_string(), key.to_string()); } - Ok(ThinLTOImportMaps { imports, exports }) + Ok(Self { keys }) } - /// Loads the ThinLTO import map from ThinLTOData. - unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImportMaps { - unsafe extern "C" fn imported_module_callback( - payload: *mut libc::c_void, - importing_module_name: *const libc::c_char, - imported_module_name: *const libc::c_char, - ) { - let map = &mut *(payload as *mut ThinLTOImportMaps); - let importing_module_name = CStr::from_ptr(importing_module_name); - let importing_module_name = module_name_to_str(&importing_module_name); - let imported_module_name = CStr::from_ptr(imported_module_name); - let imported_module_name = module_name_to_str(&imported_module_name); - - if !map.imports.contains_key(importing_module_name) { - map.imports.insert(importing_module_name.to_owned(), vec![]); - } - - map.imports - .get_mut(importing_module_name) - .unwrap() - .push(imported_module_name.to_owned()); - - if !map.exports.contains_key(imported_module_name) { - map.exports.insert(imported_module_name.to_owned(), vec![]); - } - - map.exports - .get_mut(imported_module_name) - .unwrap() - .push(importing_module_name.to_owned()); - } - - let mut map = ThinLTOImportMaps::default(); - llvm::LLVMRustGetThinLTOModuleImports( - data, - imported_module_callback, - &mut map as *mut _ as *mut libc::c_void, - ); - map + fn from_thin_lto_modules( + data: &ThinData, + modules: &[llvm::ThinLTOModule], + names: &[CString], + ) -> Self { + let keys = modules + .iter() + .zip(names.iter()) + .map(|(module, name)| { + let key = build_string(|rust_str| unsafe { + llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0); + }) + .expect("Invalid ThinLTO module key"); + (name.clone().into_string().unwrap(), key) + }) + .collect(); + Self { keys } } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1696f35563d..d3c71ff501b 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -417,7 +417,8 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { - attributes::apply_target_cpu_attr(self, llfn) + attributes::apply_target_cpu_attr(self, llfn); + attributes::apply_tune_cpu_attr(self, llfn); } fn create_used_variable(&self) { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 1237b39b300..b5f83118d0c 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -116,6 +116,9 @@ impl ExtraBackendMethods for LlvmCodegenBackend { fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { llvm_util::target_cpu(sess) } + fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { + llvm_util::tune_cpu(sess) + } } impl WriteBackendMethods for LlvmCodegenBackend { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index af3f3e7aa03..4c1fee0106a 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2362,4 +2362,10 @@ extern "C" { bytecode_len: usize, ) -> bool; pub fn LLVMRustLinkerFree(linker: &'a mut Linker<'a>); + #[allow(improper_ctypes)] + pub fn LLVMRustComputeLTOCacheKey( + key_out: &RustString, + mod_id: *const c_char, + data: &ThinLTOData, + ); } diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index d42020047fd..88d73f2a864 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -122,7 +122,7 @@ unsafe fn configure_llvm(sess: &Session) { llvm::LLVMInitializePasses(); - ::rustc_llvm::initialize_available_targets(); + rustc_llvm::initialize_available_targets(); llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); } @@ -202,11 +202,7 @@ pub(crate) fn print(req: PrintRequest, sess: &Session) { } } -pub fn target_cpu(sess: &Session) -> &str { - let name = match sess.opts.cg.target_cpu { - Some(ref s) => &**s, - None => &*sess.target.target.options.cpu, - }; +fn handle_native(name: &str) -> &str { if name != "native" { return name; } @@ -217,3 +213,19 @@ pub fn target_cpu(sess: &Session) -> &str { str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() } } + +pub fn target_cpu(sess: &Session) -> &str { + let name = match sess.opts.cg.target_cpu { + Some(ref s) => &**s, + None => &*sess.target.target.options.cpu, + }; + + handle_native(name) +} + +pub fn tune_cpu(sess: &Session) -> Option<&str> { + match sess.opts.debugging_opts.tune_cpu { + Some(ref s) => Some(handle_native(&**s)), + None => None, + } +} |