Support local ThinLTO with incremental compilation.

2 files changed, 473 insertions, 176 deletions

diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs
index 22d862f4ad5..98131349927 100644
--- a/src/librustc_codegen_llvm/back/lto.rs
+++ b/src/librustc_codegen_llvm/back/lto.rs
@@ -16,13 +16,14 @@ use errors::{FatalError, Handler};
 use llvm::archive_ro::ArchiveRO;
 use llvm::{True, False};
 use llvm;
+use memmap;
 use rustc::hir::def_id::LOCAL_CRATE;
 use rustc::middle::exported_symbols::SymbolExportLevel;
 use rustc::session::config::{self, Lto};
 use rustc::util::common::time_ext;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use time_graph::Timeline;
-use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource};
+use {ModuleCodegen, ModuleLlvm, ModuleKind};
 
 use libc;
 
@@ -82,8 +83,8 @@ impl LtoModuleCodegen {
                 let module = module.take().unwrap();
                 {
                     let config = cgcx.config(module.kind);
-                    let llmod = module.llvm().unwrap().llmod();
-                    let tm = &*module.llvm().unwrap().tm;
+                    let llmod = module.module_llvm.llmod();
+                    let tm = &*module.module_llvm.tm;
                     run_pass_manager(cgcx, tm, llmod, config, false);
                     timeline.record("fat-done");
                 }
@@ -106,6 +107,7 @@ impl LtoModuleCodegen {
 
 pub(crate) fn run(cgcx: &CodegenContext,
                   modules: Vec<ModuleCodegen>,
+                  import_only_modules: Vec<(SerializedModule, CString)>,
                   timeline: &mut Timeline)
     -> Result<Vec<LtoModuleCodegen>, FatalError>
 {
@@ -194,11 +196,19 @@ pub(crate) fn run(cgcx: &CodegenContext,
         }
     }
 
-    let arr = symbol_white_list.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
+    let symbol_white_list = symbol_white_list.iter()
+                                             .map(|c| c.as_ptr())
+                                             .collect::<Vec<_>>();
     match cgcx.lto {
         Lto::Yes | // `-C lto` == fat LTO by default
         Lto::Fat => {
-            fat_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline)
+            assert!(import_only_modules.is_empty());
+            fat_lto(cgcx,
+                    &diag_handler,
+                    modules,
+                    upstream_modules,
+                    &symbol_white_list,
+                    timeline)
         }
         Lto::Thin |
         Lto::ThinLocal => {
@@ -206,7 +216,13 @@ pub(crate) fn run(cgcx: &CodegenContext,
                 unreachable!("We should never reach this case if the LTO step \
                               is deferred to the linker");
             }
-            thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline)
+            thin_lto(cgcx,
+                     &diag_handler,
+                     modules,
+                     upstream_modules,
+                     import_only_modules,
+                     &symbol_white_list,
+                     timeline)
         }
         Lto::No => unreachable!(),
     }
@@ -236,7 +252,7 @@ fn fat_lto(cgcx: &CodegenContext,
         .filter(|&(_, module)| module.kind == ModuleKind::Regular)
         .map(|(i, module)| {
             let cost = unsafe {
-                llvm::LLVMRustModuleCost(module.llvm().unwrap().llmod())
+                llvm::LLVMRustModuleCost(module.module_llvm.llmod())
             };
             (cost, i)
         })
@@ -246,7 +262,7 @@ fn fat_lto(cgcx: &CodegenContext,
     let mut serialized_bitcode = Vec::new();
     {
         let (llcx, llmod) = {
-            let llvm = module.llvm().expect("can't lto pre-codegened modules");
+            let llvm = &module.module_llvm;
             (&llvm.llcx, llvm.llmod())
         };
         info!("using {:?} as a base module", module.name);
@@ -262,8 +278,7 @@ fn fat_lto(cgcx: &CodegenContext,
         // way we know of to do that is to serialize them to a string and them parse
         // them later. Not great but hey, that's why it's "fat" LTO, right?
         for module in modules {
-            let llvm = module.llvm().expect("can't lto pre-codegened modules");
-            let buffer = ModuleBuffer::new(llvm.llmod());
+            let buffer = ModuleBuffer::new(module.module_llvm.llmod());
             let llmod_id = CString::new(&module.name[..]).unwrap();
             serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
         }
@@ -373,6 +388,7 @@ fn thin_lto(cgcx: &CodegenContext,
             diag_handler: &Handler,
             modules: Vec<ModuleCodegen>,
             serialized_modules: Vec<(SerializedModule, CString)>,
+            import_only_modules: Vec<(SerializedModule, CString)>,
             symbol_white_list: &[*const libc::c_char],
             timeline: &mut Timeline)
     -> Result<Vec<LtoModuleCodegen>, FatalError>
@@ -393,9 +409,8 @@ fn thin_lto(cgcx: &CodegenContext,
         //        analysis!
         for (i, module) in modules.iter().enumerate() {
             info!("local module: {} - {}", i, module.name);
-            let llvm = module.llvm().expect("can't lto precodegened module");
             let name = CString::new(module.name.clone()).unwrap();
-            let buffer = ThinBuffer::new(llvm.llmod());
+            let buffer = ThinBuffer::new(module.module_llvm.llmod());
             thin_modules.push(llvm::ThinLTOModule {
                 identifier: name.as_ptr(),
                 data: buffer.data().as_ptr(),
@@ -434,6 +449,22 @@ fn thin_lto(cgcx: &CodegenContext,
             module_names.push(name);
         }
 
+        // All the modules collected up to this point we actually want to
+        // optimize. The `import_only_modules` below need to be in the list of
+        // available modules but we don't need to run optimizations for them
+        // since we already have their optimized version cached.
+        let modules_to_optimize = module_names.len();
+        for (module, name) in import_only_modules {
+            info!("foreign module {:?}", name);
+            thin_modules.push(llvm::ThinLTOModule {
+                identifier: name.as_ptr(),
+                data: module.data().as_ptr(),
+                len: module.data().len(),
+            });
+            serialized.push(module);
+            module_names.push(name);
+        }
+
         // Delegate to the C++ bindings to create some data here. Once this is a
         // tried-and-true interface we may wish to try to upstream some of this
         // to LLVM itself, right now we reimplement a lot of what they do
@@ -450,7 +481,21 @@ fn thin_lto(cgcx: &CodegenContext,
         // Save the ThinLTO import information for incremental compilation.
         if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
             let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME);
-            let imports = ThinLTOImports::from_thin_lto_data(data);
+
+            // The import information from the current compilation session. It
+            // does not contain info about modules that have been loaded from
+            // the cache instead of having been recompiled...
+            let current_imports = ThinLTOImports::from_thin_lto_data(data);
+
+            // ... so we load this additional information from the previous
+            // cache file if necessary.
+            let imports = if path.exists() {
+                let prev_imports = ThinLTOImports::load_from_file(&path).unwrap();
+                prev_imports.update(current_imports, &module_names)
+            } else {
+                current_imports
+            };
+
             if let Err(err) = imports.save_to_file(&path) {
                 let msg = format!("Error while writing ThinLTO import data: {}",
                                   err);
@@ -472,7 +517,7 @@ fn thin_lto(cgcx: &CodegenContext,
             serialized_modules: serialized,
             module_names,
         });
-        Ok((0..shared.module_names.len()).map(|i| {
+        Ok((0..modules_to_optimize).map(|i| {
             LtoModuleCodegen::Thin(ThinModule {
                 shared: shared.clone(),
                 idx: i,
@@ -546,6 +591,7 @@ fn run_pass_manager(cgcx: &CodegenContext,
 pub enum SerializedModule {
     Local(ModuleBuffer),
     FromRlib(Vec<u8>),
+    FromUncompressedFile(memmap::Mmap, File),
 }
 
 impl SerializedModule {
@@ -553,6 +599,7 @@ impl SerializedModule {
         match *self {
             SerializedModule::Local(ref m) => m.data(),
             SerializedModule::FromRlib(ref m) => m,
+            SerializedModule::FromUncompressedFile(ref m, _) => m,
         }
     }
 }
@@ -682,16 +729,16 @@ impl ThinModule {
             write::llvm_err(&diag_handler, msg)
         })? as *const _;
         let module = ModuleCodegen {
-            source: ModuleSource::Codegened(ModuleLlvm {
+            module_llvm: ModuleLlvm {
                 llmod_raw,
                 llcx,
                 tm,
-            }),
+            },
             name: self.name().to_string(),
             kind: ModuleKind::Regular,
         };
         {
-            let llmod = module.llvm().unwrap().llmod();
+            let llmod = module.module_llvm.llmod();
             cgcx.save_temp_bitcode(&module, "thin-lto-input");
 
             // Before we do much else find the "main" `DICompileUnit` that we'll be
@@ -787,7 +834,7 @@ impl ThinModule {
             // little differently.
             info!("running thin lto passes over {}", module.name);
             let config = cgcx.config(module.kind);
-            run_pass_manager(cgcx, module.llvm().unwrap().tm, llmod, config, true);
+            run_pass_manager(cgcx, module.module_llvm.tm, llmod, config, true);
             cgcx.save_temp_bitcode(&module, "thin-lto-after-pm");
             timeline.record("thin-done");
         }
@@ -809,6 +856,26 @@ impl ThinLTOImports {
         }
     }
 
+    pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] {
+        self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[])
+    }
+
+    pub fn update(mut self, new: ThinLTOImports, module_names: &[CString]) -> ThinLTOImports {
+        let module_names: FxHashSet<_> = module_names.iter().map(|name| {
+            name.clone().into_string().unwrap()
+        }).collect();
+
+        // Remove all modules that don't exist anymore.
+        self.imports.retain(|k, _| module_names.contains(k));
+
+        // Overwrite old values
+        for (importing_module, imported_modules) in new.imports {
+            self.imports.insert(importing_module, imported_modules);
+        }
+
+        self
+    }
+
     /// Load the ThinLTO import map from ThinLTOData.
     unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports {
         fn module_name_to_str(c_str: &CStr) -> &str {
@@ -832,6 +899,7 @@ impl ThinLTOImports {
             if !map.imports.contains_key(importing_module_name) {
                 map.imports.insert(importing_module_name.to_owned(), vec![]);
             }
+
             map.imports
                .get_mut(importing_module_name)
                .unwrap()
@@ -888,4 +956,4 @@ impl ThinLTOImports {
             imports
         })
     }
-}
\ No newline at end of file
+}
diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs
index 2373428d68c..e1d69db83b9 100644
--- a/src/librustc_codegen_llvm/back/write.rs
+++ b/src/librustc_codegen_llvm/back/write.rs
@@ -10,14 +10,16 @@
 
 use attributes;
 use back::bytecode::{self, RLIB_BYTECODE_EXTENSION};
-use back::lto::{self, ModuleBuffer, ThinBuffer};
+use back::lto::{self, ModuleBuffer, ThinBuffer, SerializedModule};
 use back::link::{self, get_linker, remove};
 use back::command::Command;
 use back::linker::LinkerInfo;
 use back::symbol_export::ExportedSymbols;
 use base;
 use consts;
-use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir, in_incr_comp_dir};
+use memmap;
+use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir,
+                        in_incr_comp_dir, in_incr_comp_dir_sess};
 use rustc::dep_graph::{WorkProduct, WorkProductId, WorkProductFileKind};
 use rustc::middle::cstore::EncodedMetadata;
 use rustc::session::config::{self, OutputFilenames, OutputType, Passes, Sanitizer, Lto};
@@ -26,7 +28,8 @@ use rustc::util::nodemap::FxHashMap;
 use time_graph::{self, TimeGraph, Timeline};
 use llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic};
 use llvm_util;
-use {CodegenResults, ModuleSource, ModuleCodegen, CompiledModule, ModuleKind};
+use {CodegenResults, ModuleCodegen, CompiledModule, ModuleKind, ModuleLlvm,
+     CachedModuleCodegen};
 use CrateInfo;
 use rustc::hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc::ty::TyCtxt;
@@ -84,6 +87,8 @@ pub const TLS_MODEL_ARGS : [(&'static str, llvm::ThreadLocalMode); 4] = [
     ("local-exec", llvm::ThreadLocalMode::LocalExec),
 ];
 
+const PRE_THIN_LTO_BC_EXT: &str = "pre-thin-lto.bc";
+
 pub fn llvm_err(handler: &errors::Handler, msg: String) -> FatalError {
     match llvm::last_error() {
         Some(err) => handler.fatal(&format!("{}: {}", msg, err)),
@@ -224,6 +229,7 @@ pub struct ModuleConfig {
 
     // Flags indicating which outputs to produce.
     emit_no_opt_bc: bool,
+    emit_pre_thin_lto_bc: bool,
     emit_bc: bool,
     emit_bc_compressed: bool,
     emit_lto_bc: bool,
@@ -260,6 +266,7 @@ impl ModuleConfig {
             pgo_use: String::new(),
 
             emit_no_opt_bc: false,
+            emit_pre_thin_lto_bc: false,
             emit_bc: false,
             emit_bc_compressed: false,
             emit_lto_bc: false,
@@ -392,7 +399,7 @@ impl CodegenContext {
             let cgu = Some(&module.name[..]);
             let path = self.output_filenames.temp_path_ext(&ext, cgu);
             let cstr = path2cstr(&path);
-            let llmod = module.llvm().unwrap().llmod();
+            let llmod = module.module_llvm.llmod();
             llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr());
         }
     }
@@ -495,13 +502,9 @@ unsafe fn optimize(cgcx: &CodegenContext,
                    timeline: &mut Timeline)
     -> Result<(), FatalError>
 {
-    let (llmod, llcx, tm) = match module.source {
-        ModuleSource::Codegened(ref llvm) => (llvm.llmod(), &*llvm.llcx, &*llvm.tm),
-        ModuleSource::Preexisting(_) => {
-            bug!("optimize_and_codegen: called with ModuleSource::Preexisting")
-        }
-    };
-
+    let llmod = module.module_llvm.llmod();
+    let llcx = &*module.module_llvm.llcx;
+    let tm = &*module.module_llvm.tm;
     let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx);
 
     let module_name = module.name.clone();
@@ -622,12 +625,20 @@ unsafe fn optimize(cgcx: &CodegenContext,
         // Deallocate managers that we're now done with
         llvm::LLVMDisposePassManager(fpm);
         llvm::LLVMDisposePassManager(mpm);
+
+        if config.emit_pre_thin_lto_bc {
+            let out = cgcx.output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT,
+                                                          module_name);
+            let out = path2cstr(&out);
+            llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr());
+        }
     }
     Ok(())
 }
 
 fn generate_lto_work(cgcx: &CodegenContext,
-                     modules: Vec<ModuleCodegen>)
+                     modules: Vec<ModuleCodegen>,
+                     import_only_modules: Vec<(SerializedModule, CString)>)
     -> Vec<(WorkItem, u64)>
 {
     let mut timeline = cgcx.time_graph.as_ref().map(|tg| {
@@ -635,7 +646,7 @@ fn generate_lto_work(cgcx: &CodegenContext,
                  CODEGEN_WORK_PACKAGE_KIND,
                  "generate lto")
     }).unwrap_or(Timeline::noop());
-    let lto_modules = lto::run(cgcx, modules, &mut timeline)
+    let lto_modules = lto::run(cgcx, modules, import_only_modules, &mut timeline)
         .unwrap_or_else(|e| e.raise());
 
     lto_modules.into_iter().map(|module| {
@@ -653,12 +664,9 @@ unsafe fn codegen(cgcx: &CodegenContext,
 {
     timeline.record("codegen");
     {
-        let (llmod, llcx, tm) = match module.source {
-            ModuleSource::Codegened(ref llvm) => (llvm.llmod(), &*llvm.llcx, &*llvm.tm),
-            ModuleSource::Preexisting(_) => {
-                bug!("codegen: called with ModuleSource::Preexisting")
-            }
-        };
+        let llmod = module.module_llvm.llmod();
+        let llcx = &*module.module_llvm.llcx;
+        let tm = &*module.module_llvm.tm;
         let module_name = module.name.clone();
         let module_name = Some(&module_name[..]);
         let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx);
@@ -912,6 +920,20 @@ fn need_crate_bitcode_for_rlib(sess: &Session) -> bool {
     sess.opts.output_types.contains_key(&OutputType::Exe)
 }
 
+fn need_pre_thin_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
+    if sess.opts.incremental.is_none() {
+        return false
+    }
+
+    match sess.lto() {
+        Lto::Yes |
+        Lto::Fat |
+        Lto::No => false,
+        Lto::Thin |
+        Lto::ThinLocal => true,
+    }
+}
+
 pub fn start_async_codegen(tcx: TyCtxt,
                                time_graph: Option<TimeGraph>,
                                metadata: EncodedMetadata,
@@ -970,6 +992,7 @@ pub fn start_async_codegen(tcx: TyCtxt,
     // Save all versions of the bytecode if we're saving our temporaries.
     if sess.opts.cg.save_temps {
         modules_config.emit_no_opt_bc = true;
+        modules_config.emit_pre_thin_lto_bc = true;
         modules_config.emit_bc = true;
         modules_config.emit_lto_bc = true;
         metadata_config.emit_bc = true;
@@ -984,6 +1007,9 @@ pub fn start_async_codegen(tcx: TyCtxt,
         allocator_config.emit_bc_compressed = true;
     }
 
+    modules_config.emit_pre_thin_lto_bc =
+        need_pre_thin_lto_bitcode_for_incr_comp(sess);
+
     modules_config.no_integrated_as = tcx.sess.opts.cg.no_integrated_as ||
         tcx.sess.target.target.options.no_integrated_as;
 
@@ -1056,7 +1082,8 @@ pub fn start_async_codegen(tcx: TyCtxt,
 
 fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
     sess: &Session,
-    compiled_modules: &CompiledModules
+    compiled_modules: &CompiledModules,
+    output_filenames: &OutputFilenames,
 ) -> FxHashMap<WorkProductId, WorkProduct> {
     let mut work_products = FxHashMap::default();
 
@@ -1064,7 +1091,7 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
         return work_products;
     }
 
-    for module in compiled_modules.modules.iter() {
+    for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
         let mut files = vec![];
 
         if let Some(ref path) = module.object {
@@ -1077,6 +1104,13 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
             files.push((WorkProductFileKind::BytecodeCompressed, path.clone()));
         }
 
+        let pre_thin_lto_bytecode_path =
+            output_filenames.temp_path_ext(PRE_THIN_LTO_BC_EXT, Some(&module.name));
+
+        if pre_thin_lto_bytecode_path.exists() {
+            files.push((WorkProductFileKind::PreThinLtoBytecode, pre_thin_lto_bytecode_path));
+        }
+
         if let Some((id, product)) =
                 copy_cgu_workproducts_to_incr_comp_cache_dir(sess, &module.name, &files) {
             work_products.insert(id, product);
@@ -1236,21 +1270,34 @@ fn produce_final_output_artifacts(sess: &Session,
     // These are used in linking steps and will be cleaned up afterward.
 }
 
-pub(crate) fn dump_incremental_data(codegen_results: &CodegenResults) {
-    println!("[incremental] Re-using {} out of {} modules",
-              codegen_results.modules.iter().filter(|m| m.pre_existing).count(),
-              codegen_results.modules.len());
+pub(crate) fn dump_incremental_data(_codegen_results: &CodegenResults) {
+    // FIXME(mw): This does not work at the moment because the situation has
+    //            become more complicated due to incremental LTO. Now a CGU
+    //            can have more than two caching states.
+    // println!("[incremental] Re-using {} out of {} modules",
+    //           codegen_results.modules.iter().filter(|m| m.pre_existing).count(),
+    //           codegen_results.modules.len());
 }
 
 enum WorkItem {
+    /// Optimize a newly codegened, totally unoptimized module.
     Optimize(ModuleCodegen),
+    /// Copy the post-LTO artifacts from the incremental cache to the output
+    /// directory.
+    CopyPostLtoArtifacts(CachedModuleCodegen),
+    /// Load the pre-LTO version of a module from the incremental cache, so it
+    /// can be run through LTO again.
+    LoadPreLtoModule(CachedModuleCodegen),
+    /// Perform (Thin)LTO on the given module.
     LTO(lto::LtoModuleCodegen),
 }
 
 impl WorkItem {
-    fn kind(&self) -> ModuleKind {
+    fn module_kind(&self) -> ModuleKind {
         match *self {
             WorkItem::Optimize(ref m) => m.kind,
+            WorkItem::CopyPostLtoArtifacts(_) |
+            WorkItem::LoadPreLtoModule(_) |
             WorkItem::LTO(_) => ModuleKind::Regular,
         }
     }
@@ -1258,6 +1305,8 @@ impl WorkItem {
     fn name(&self) -> String {
         match *self {
             WorkItem::Optimize(ref m) => format!("optimize: {}", m.name),
+            WorkItem::LoadPreLtoModule(ref m) => format!("load pre-lto module: {}", m.name),
+            WorkItem::CopyPostLtoArtifacts(ref m) => format!("copy post LTO artifacts: {}", m.name),
             WorkItem::LTO(ref m) => format!("lto: {}", m.name()),
         }
     }
@@ -1273,141 +1322,254 @@ fn execute_work_item(cgcx: &CodegenContext,
                      timeline: &mut Timeline)
     -> Result<WorkItemResult, FatalError>
 {
+    match work_item {
+        work_item @ WorkItem::Optimize(_) => {
+            execute_optimize_work_item(cgcx, work_item, timeline)
+        }
+        work_item @ WorkItem::LoadPreLtoModule(_) => {
+            execute_load_pre_lto_mod_work_item(cgcx, work_item, timeline)
+        }
+        work_item @ WorkItem::CopyPostLtoArtifacts(_) => {
+            execute_copy_from_cache_work_item(cgcx, work_item, timeline)
+        }
+        work_item @ WorkItem::LTO(_) => {
+            execute_lto_work_item(cgcx, work_item, timeline)
+        }
+    }
+}
+
+fn execute_optimize_work_item(cgcx: &CodegenContext,
+                              work_item: WorkItem,
+                              timeline: &mut Timeline)
+    -> Result<WorkItemResult, FatalError>
+{
+    let config = cgcx.config(work_item.module_kind());
+
+    let module = if let WorkItem::Optimize(module) = work_item {
+        module
+    } else {
+        bug!("execute_optimize_work_item() called with non-WorkItem::Optimize");
+    };
+
     let diag_handler = cgcx.create_diag_handler();
-    let config = cgcx.config(work_item.kind());
-    let module = match work_item {
-        WorkItem::Optimize(module) => module,
-        WorkItem::LTO(mut lto) => {
-            unsafe {
-                let module = lto.optimize(cgcx, timeline)?;
-                let module = codegen(cgcx, &diag_handler, module, config, timeline)?;
-                return Ok(WorkItemResult::Compiled(module))
-            }
+
+    unsafe {
+        optimize(cgcx, &diag_handler, &module, config, timeline)?;
+    }
+
+    let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled();
+
+    // After we've done the initial round of optimizations we need to
+    // decide whether to synchronously codegen this module or ship it
+    // back to the coordinator thread for further LTO processing (which
+    // has to wait for all the initial modules to be optimized).
+    //
+    // Here we dispatch based on the `cgcx.lto` and kind of module we're
+    // codegenning...
+    let needs_lto = match cgcx.lto {
+        Lto::No => false,
+
+        // If the linker does LTO, we don't have to do it. Note that we
+        // keep doing full LTO, if it is requested, as not to break the
+        // assumption that the output will be a single module.
+        Lto::Thin | Lto::ThinLocal if linker_does_lto => false,
+
+        // Here we've got a full crate graph LTO requested. We ignore
+        // this, however, if the crate type is only an rlib as there's
+        // no full crate graph to process, that'll happen later.
+        //
+        // This use case currently comes up primarily for targets that
+        // require LTO so the request for LTO is always unconditionally
+        // passed down to the backend, but we don't actually want to do
+        // anything about it yet until we've got a final product.
+        Lto::Yes | Lto::Fat | Lto::Thin => {
+            cgcx.crate_types.len() != 1 ||
+                cgcx.crate_types[0] != config::CrateType::Rlib
+        }
+
+        // When we're automatically doing ThinLTO for multi-codegen-unit
+        // builds we don't actually want to LTO the allocator modules if
+        // it shows up. This is due to various linker shenanigans that
+        // we'll encounter later.
+        //
+        // Additionally here's where we also factor in the current LLVM
+        // version. If it doesn't support ThinLTO we skip this.
+        Lto::ThinLocal => {
+            module.kind != ModuleKind::Allocator &&
+                unsafe { llvm::LLVMRustThinLTOAvailable() }
         }
     };
-    let module_name = module.name.clone();
 
-    let pre_existing = match module.source {
-        ModuleSource::Codegened(_) => None,
-        ModuleSource::Preexisting(ref wp) => Some(wp.clone()),
+    // Metadata modules never participate in LTO regardless of the lto
+    // settings.
+    let needs_lto = needs_lto && module.kind != ModuleKind::Metadata;
+
+    if needs_lto {
+        Ok(WorkItemResult::NeedsLTO(module))
+    } else {
+        let module = unsafe {
+            codegen(cgcx, &diag_handler, module, config, timeline)?
+        };
+        Ok(WorkItemResult::Compiled(module))
+    }
+}
+
+fn execute_copy_from_cache_work_item(cgcx: &CodegenContext,
+                                     work_item: WorkItem,
+                                     _: &mut Timeline)
+    -> Result<WorkItemResult, FatalError>
+{
+    let config = cgcx.config(work_item.module_kind());
+
+    let module = if let WorkItem::CopyPostLtoArtifacts(module) = work_item {
+        module
+    } else {
+        bug!("execute_copy_from_cache_work_item() called with wrong WorkItem kind.")
     };
 
-    if let Some(wp) = pre_existing {
-        let incr_comp_session_dir = cgcx.incr_comp_session_dir
-                                        .as_ref()
-                                        .unwrap();
-        let name = &module.name;
-        let mut object = None;
-        let mut bytecode = None;
-        let mut bytecode_compressed = None;
-        for (kind, saved_file) in wp.saved_files {
-            let obj_out = match kind {
-                WorkProductFileKind::Object => {
-                    let path = cgcx.output_filenames.temp_path(OutputType::Object, Some(name));
-                    object = Some(path.clone());
-                    path
-                }
-                WorkProductFileKind::Bytecode => {
-                    let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name));
-                    bytecode = Some(path.clone());
-                    path
-                }
-                WorkProductFileKind::BytecodeCompressed => {
-                    let path = cgcx.output_filenames.temp_path(OutputType::Bitcode, Some(name))
-                        .with_extension(RLIB_BYTECODE_EXTENSION);
-                    bytecode_compressed = Some(path.clone());
-                    path
-                }
-            };
-            let source_file = in_incr_comp_dir(&incr_comp_session_dir,
-                                               &saved_file);
-            debug!("copying pre-existing module `{}` from {:?} to {}",
-                   module.name,
-                   source_file,
-                   obj_out.display());
-            match link_or_copy(&source_file, &obj_out) {
-                Ok(_) => { }
-                Err(err) => {
-                    diag_handler.err(&format!("unable to copy {} to {}: {}",
-                                              source_file.display(),
-                                              obj_out.display(),
-                                              err));
-                }
+    let incr_comp_session_dir = cgcx.incr_comp_session_dir
+                                    .as_ref()
+                                    .unwrap();
+    let mut object = None;
+    let mut bytecode = None;
+    let mut bytecode_compressed = None;
+    for (kind, saved_file) in &module.source.saved_files {
+        let obj_out = match kind {
+            WorkProductFileKind::Object => {
+                let path = cgcx.output_filenames.temp_path(OutputType::Object,
+                                                           Some(&module.name));
+                object = Some(path.clone());
+                path
+            }
+            WorkProductFileKind::Bytecode => {
+                let path = cgcx.output_filenames.temp_path(OutputType::Bitcode,
+                                                           Some(&module.name));
+                bytecode = Some(path.clone());
+                path
+            }
+            WorkProductFileKind::BytecodeCompressed => {
+                let path = cgcx.output_filenames.temp_path(OutputType::Bitcode,
+                                                           Some(&module.name))
+                    .with_extension(RLIB_BYTECODE_EXTENSION);
+                bytecode_compressed = Some(path.clone());
+                path
+            }
+            WorkProductFileKind::PreThinLtoBytecode => {
+                continue;
+            }
+        };
+        let source_file = in_incr_comp_dir(&incr_comp_session_dir,
+                                           &saved_file);
+        debug!("copying pre-existing module `{}` from {:?} to {}",
+               module.name,
+               source_file,
+               obj_out.display());
+        match link_or_copy(&source_file, &obj_out) {
+            Ok(_) => { }
+            Err(err) => {
+                let diag_handler = cgcx.create_diag_handler();
+                diag_handler.err(&format!("unable to copy {} to {}: {}",
+                                          source_file.display(),
+                                          obj_out.display(),
+                                          err));
             }
         }
-        assert_eq!(object.is_some(), config.emit_obj);
-        assert_eq!(bytecode.is_some(), config.emit_bc);
-        assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed);
-
-        Ok(WorkItemResult::Compiled(CompiledModule {
-            name: module_name,
-            kind: ModuleKind::Regular,
-            pre_existing: true,
-            object,
-            bytecode,
-            bytecode_compressed,
-        }))
-    } else {
-        debug!("llvm-optimizing {:?}", module_name);
+    }
+
+    assert_eq!(object.is_some(), config.emit_obj);
+    assert_eq!(bytecode.is_some(), config.emit_bc);
+    assert_eq!(bytecode_compressed.is_some(), config.emit_bc_compressed);
+
+    Ok(WorkItemResult::Compiled(CompiledModule {
+        name: module.name,
+        kind: ModuleKind::Regular,
+        object,
+        bytecode,
+        bytecode_compressed,
+    }))
+}
+
+fn execute_lto_work_item(cgcx: &CodegenContext,
+                         work_item: WorkItem,
+                         timeline: &mut Timeline)
+    -> Result<WorkItemResult, FatalError>
+{
+    let config = cgcx.config(work_item.module_kind());
+
+    if let WorkItem::LTO(mut lto) = work_item {
+        let diag_handler = cgcx.create_diag_handler();
 
         unsafe {
-            optimize(cgcx, &diag_handler, &module, config, timeline)?;
-
-            let linker_does_lto = cgcx.opts.debugging_opts.cross_lang_lto.enabled();
-
-            // After we've done the initial round of optimizations we need to
-            // decide whether to synchronously codegen this module or ship it
-            // back to the coordinator thread for further LTO processing (which
-            // has to wait for all the initial modules to be optimized).
-            //
-            // Here we dispatch based on the `cgcx.lto` and kind of module we're
-            // codegenning...
-            let needs_lto = match cgcx.lto {
-                Lto::No => false,
-
-                // If the linker does LTO, we don't have to do it. Note that we
-                // keep doing full LTO, if it is requested, as not to break the
-                // assumption that the output will be a single module.
-                Lto::Thin | Lto::ThinLocal if linker_does_lto => false,
-
-                // Here we've got a full crate graph LTO requested. We ignore
-                // this, however, if the crate type is only an rlib as there's
-                // no full crate graph to process, that'll happen later.
-                //
-                // This use case currently comes up primarily for targets that
-                // require LTO so the request for LTO is always unconditionally
-                // passed down to the backend, but we don't actually want to do
-                // anything about it yet until we've got a final product.
-                Lto::Yes | Lto::Fat | Lto::Thin => {
-                    cgcx.crate_types.len() != 1 ||
-                        cgcx.crate_types[0] != config::CrateType::Rlib
-                }
+            let module = lto.optimize(cgcx, timeline)?;
+            let module = codegen(cgcx, &diag_handler, module, config, timeline)?;
+            Ok(WorkItemResult::Compiled(module))
+        }
+    } else {
+        bug!("execute_lto_work_item() called with wrong WorkItem kind.")
+    }
+}
 
-                // When we're automatically doing ThinLTO for multi-codegen-unit
-                // builds we don't actually want to LTO the allocator modules if
-                // it shows up. This is due to various linker shenanigans that
-                // we'll encounter later.
-                //
-                // Additionally here's where we also factor in the current LLVM
-                // version. If it doesn't support ThinLTO we skip this.
-                Lto::ThinLocal => {
-                    module.kind != ModuleKind::Allocator &&
-                        llvm::LLVMRustThinLTOAvailable()
-                }
-            };
+fn execute_load_pre_lto_mod_work_item(cgcx: &CodegenContext,
+                                      work_item: WorkItem,
+                                      _: &mut Timeline)
+    -> Result<WorkItemResult, FatalError>
+{
+    let module = if let WorkItem::LoadPreLtoModule(module) = work_item {
+        module
+    } else {
+        bug!("execute_load_pre_lto_mod_work_item() called with wrong WorkItem kind.")
+    };
 
-            // Metadata modules never participate in LTO regardless of the lto
-            // settings.
-            let needs_lto = needs_lto && module.kind != ModuleKind::Metadata;
+    let work_product = module.source.clone();
+    let incr_comp_session_dir = cgcx.incr_comp_session_dir
+                                    .as_ref()
+                                    .unwrap();
 
-            if needs_lto {
-                Ok(WorkItemResult::NeedsLTO(module))
-            } else {
-                let module = codegen(cgcx, &diag_handler, module, config, timeline)?;
-                Ok(WorkItemResult::Compiled(module))
+    let filename = pre_lto_bitcode_filename(&work_product);
+    let bc_path = in_incr_comp_dir(&incr_comp_session_dir, &filename);
+
+    let file = fs::File::open(&bc_path).unwrap_or_else(|e| {
+        panic!("failed to open bitcode file `{}`: {}",
+                           bc_path.display(),
+                           e);
+    });
+
+    let module_llvm = unsafe {
+        let data = ::memmap::Mmap::map(&file).unwrap_or_else(|e| {
+            panic!("failed to create mmap for bitcode file `{}`: {}",
+                               bc_path.display(),
+                               e);
+        });
+
+        let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
+        let mod_name_c = SmallCStr::new(&module.name);
+        let llmod_raw = match llvm::LLVMRustParseBitcodeForThinLTO(
+            llcx,
+            data.as_ptr(),
+            data.len(),
+            mod_name_c.as_ptr(),
+        ) {
+            Some(m) => m as *const _,
+            None => {
+                panic!("failed to parse bitcode for thin LTO module `{}`",
+                        module.name);
             }
+        };
+
+        let tm = (cgcx.tm_factory)().unwrap();
+
+        ModuleLlvm {
+            llmod_raw,
+            llcx,
+            tm,
         }
-    }
+    };
+
+    Ok(WorkItemResult::NeedsLTO(ModuleCodegen {
+        name: module.name.to_string(),
+        module_llvm,
+        kind: ModuleKind::Regular,
+    }))
 }
 
 enum Message {
@@ -1424,6 +1586,10 @@ enum Message {
         llvm_work_item: WorkItem,
         cost: u64,
     },
+    AddImportOnlyModule {
+        module_data: SerializedModule,
+        module_name: CString,
+    },
     CodegenComplete,
     CodegenItem,
 }
@@ -1703,6 +1869,7 @@ fn start_executing_work(tcx: TyCtxt,
         let mut compiled_metadata_module = None;
         let mut compiled_allocator_module = None;
         let mut needs_lto = Vec::new();
+        let mut lto_import_only_modules = Vec::new();
         let mut started_lto = false;
 
         // This flag tracks whether all items have gone through codegens
@@ -1749,7 +1916,7 @@ fn start_executing_work(tcx: TyCtxt,
                             worker: get_worker_id(&mut free_worker_ids),
                             .. cgcx.clone()
                         };
-                        maybe_start_llvm_timer(cgcx.config(item.kind()),
+                        maybe_start_llvm_timer(cgcx.config(item.module_kind()),
                                                &mut llvm_start_time);
                         main_thread_worker_state = MainThreadWorkerState::LLVMing;
                         spawn_work(cgcx, item);
@@ -1768,7 +1935,9 @@ fn start_executing_work(tcx: TyCtxt,
                     assert!(needs_lto.len() > 0);
                     started_lto = true;
                     let modules = mem::replace(&mut needs_lto, Vec::new());
-                    for (work, cost) in generate_lto_work(&cgcx, modules) {
+                    let import_only_modules =
+                        mem::replace(&mut lto_import_only_modules, Vec::new());
+                    for (work, cost) in generate_lto_work(&cgcx, modules, import_only_modules) {
                         let insertion_index = work_items
                             .binary_search_by_key(&cost, |&(_, cost)| cost)
                             .unwrap_or_else(|e| e);
@@ -1789,7 +1958,7 @@ fn start_executing_work(tcx: TyCtxt,
                                 worker: get_worker_id(&mut free_worker_ids),
                                 .. cgcx.clone()
                             };
-                            maybe_start_llvm_timer(cgcx.config(item.kind()),
+                            maybe_start_llvm_timer(cgcx.config(item.module_kind()),
                                                    &mut llvm_start_time);
                             main_thread_worker_state = MainThreadWorkerState::LLVMing;
                             spawn_work(cgcx, item);
@@ -1820,7 +1989,7 @@ fn start_executing_work(tcx: TyCtxt,
             while work_items.len() > 0 && running < tokens.len() {
                 let (item, _) = work_items.pop().unwrap();
 
-                maybe_start_llvm_timer(cgcx.config(item.kind()),
+                maybe_start_llvm_timer(cgcx.config(item.module_kind()),
                                        &mut llvm_start_time);
 
                 let cgcx = CodegenContext {
@@ -1932,10 +2101,14 @@ fn start_executing_work(tcx: TyCtxt,
                     } else {
                         running -= 1;
                     }
-
                     free_worker_ids.push(worker_id);
                     needs_lto.push(result);
                 }
+                Message::AddImportOnlyModule { module_data, module_name } => {
+                    assert!(!started_lto);
+                    assert!(!codegen_done);
+                    lto_import_only_modules.push((module_data, module_name));
+                }
                 Message::Done { result: Err(()), worker_id: _ } => {
                     shared_emitter.fatal("aborting due to worker thread failure");
                     // Exit the coordinator thread
@@ -2308,9 +2481,10 @@ impl OngoingCodegen {
             time_graph.dump(&format!("{}-timings", self.crate_name));
         }
 
-        let work_products = copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess,
-                                                                             &compiled_modules);
-
+        let work_products =
+            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess,
+                                                             &compiled_modules,
+                                                             &self.output_filenames);
         produce_final_output_artifacts(sess,
                                        &compiled_modules,
                                        &self.output_filenames);
@@ -2371,8 +2545,8 @@ impl OngoingCodegen {
 }
 
 pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt,
-                                               module: ModuleCodegen,
-                                               cost: u64) {
+                                              module: ModuleCodegen,
+                                              cost: u64) {
     let llvm_work_item = WorkItem::Optimize(module);
     drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone {
         llvm_work_item,
@@ -2380,6 +2554,61 @@ pub(crate) fn submit_codegened_module_to_llvm(tcx: TyCtxt,
     })));
 }
 
+pub(crate) fn submit_post_lto_module_to_llvm(tcx: TyCtxt,
+                                             module: CachedModuleCodegen) {
+    let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
+    drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone {
+        llvm_work_item,
+        cost: 0,
+    })));
+}
+
+pub(crate) fn submit_pre_lto_module_to_llvm(tcx: TyCtxt,
+                                            module: CachedModuleCodegen) {
+    let llvm_work_item = WorkItem::LoadPreLtoModule(module);
+
+    drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::CodegenDone {
+        llvm_work_item,
+        // We don't know the size of the module, but just loading will have smaller
+        // cost than optimizing.
+        cost: 10,
+    })));
+}
+
+pub(crate) fn submit_import_only_module_to_llvm(tcx: TyCtxt,
+                                                module: CachedModuleCodegen) {
+    let filename = pre_lto_bitcode_filename(&module.source);
+    let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
+    let file = fs::File::open(&bc_path).unwrap_or_else(|e| {
+        panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e)
+    });
+
+    let mmap = unsafe {
+        memmap::Mmap::map(&file).unwrap_or_else(|e| {
+            panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
+        })
+    };
+
+    // Schedule the module to be loaded
+    drop(tcx.tx_to_llvm_workers.lock().send(Box::new(Message::AddImportOnlyModule {
+        module_data: SerializedModule::FromUncompressedFile(mmap, file),
+        module_name: CString::new(module.name.clone()).unwrap(),
+    })));
+
+    // Note: We also schedule for the cached files to be copied to the output
+    // directory
+    submit_post_lto_module_to_llvm(tcx, module);
+}
+
+fn pre_lto_bitcode_filename(wp: &WorkProduct) -> String {
+    wp.saved_files
+      .iter()
+      .find(|&&(kind, _)| kind == WorkProductFileKind::PreThinLtoBytecode)
+      .map(|&(_, ref filename)| filename.clone())
+      .unwrap_or_else(|| panic!("Couldn't find pre-thin-lto bytecode for `{}`",
+                                wp.cgu_name))
+}
+
 fn msvc_imps_needed(tcx: TyCtxt) -> bool {
     // This should never be true (because it's not supported). If it is true,
     // something is wrong with commandline arg validation.