Auto merge of #54325 - michaelwoerister:incr-thinlto-tests, r=alexcrichton

incr.comp.: Allow for more fine-grained testing of CGU reuse and use it to test incremental ThinLTO.

This adds some tests specifically targeted at incremental ThinLTO, plus the infrastructure for tracking the kind of cache hit/miss we had for a given CGU. @alexcrichton, let me know if you can think of any more tests to add. ThinLTO works rather reliably for small functions, so we should be able to test it in a robust way.

I think after this lands it might time for a "Help us test incremental ThinLTO" post on irlo.

r? @alexcrichton

3 files changed, 24 insertions, 26 deletions

diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs
index 364b469738f..3ac22f4eaef 100644
--- a/src/librustc_codegen_llvm/back/lto.rs
+++ b/src/librustc_codegen_llvm/back/lto.rs
@@ -18,6 +18,7 @@ use llvm::{True, False};
 use llvm;
 use memmap;
 use rustc::dep_graph::WorkProduct;
+use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
 use rustc::hir::def_id::LOCAL_CRATE;
 use rustc::middle::exported_symbols::SymbolExportLevel;
 use rustc::session::config::{self, Lto};
@@ -538,6 +539,8 @@ fn thin_lto(cgcx: &CodegenContext,
                     let work_product = green_modules[module_name].clone();
                     copy_jobs.push(work_product);
                     info!(" - {}: re-used", module_name);
+                    cgcx.cgu_reuse_tracker.set_actual_reuse(module_name,
+                                                            CguReuse::PostLto);
                     continue
                 }
             }
diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs
index 7b78d4fb4ff..447b505e79c 100644
--- a/src/librustc_codegen_llvm/back/write.rs
+++ b/src/librustc_codegen_llvm/back/write.rs
@@ -21,6 +21,7 @@ use memmap;
 use rustc_incremental::{copy_cgu_workproducts_to_incr_comp_cache_dir,
                         in_incr_comp_dir, in_incr_comp_dir_sess};
 use rustc::dep_graph::{WorkProduct, WorkProductId, WorkProductFileKind};
+use rustc::dep_graph::cgu_reuse_tracker::CguReuseTracker;
 use rustc::middle::cstore::EncodedMetadata;
 use rustc::session::config::{self, OutputFilenames, OutputType, Passes, Sanitizer, Lto};
 use rustc::session::Session;
@@ -377,6 +378,8 @@ pub struct CodegenContext {
     // The incremental compilation session directory, or None if we are not
     // compiling incrementally
     pub incr_comp_session_dir: Option<PathBuf>,
+    // Used to update CGU re-use information during the thinlto phase.
+    pub cgu_reuse_tracker: CguReuseTracker,
     // Channel back to the main control thread to send messages to
     coordinator_send: Sender<Box<dyn Any + Send>>,
     // A reference to the TimeGraph so we can register timings. None means that
@@ -1607,6 +1610,7 @@ fn start_executing_work(tcx: TyCtxt,
         remark: sess.opts.cg.remark.clone(),
         worker: 0,
         incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
+        cgu_reuse_tracker: sess.cgu_reuse_tracker.clone(),
         coordinator_send,
         diag_emitter: shared_emitter.clone(),
         time_graph,
@@ -2390,6 +2394,8 @@ impl OngoingCodegen {
             }
         };
 
+        sess.cgu_reuse_tracker.check_expected_reuse(sess);
+
         sess.abort_if_errors();
 
         if let Some(time_graph) = self.time_graph {
diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs
index c1f6006e684..55dc43ec1fb 100644
--- a/src/librustc_codegen_llvm/base.rs
+++ b/src/librustc_codegen_llvm/base.rs
@@ -32,6 +32,7 @@ use abi;
 use back::write::{self, OngoingCodegen};
 use llvm::{self, TypeKind, get_param};
 use metadata;
+use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
 use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE};
 use rustc::middle::lang_items::StartFnLangItem;
 use rustc::middle::weak_lang_items;
@@ -697,25 +698,18 @@ pub fn iter_globals(llmod: &'ll llvm::Module) -> ValueIter<'ll> {
     }
 }
 
-#[derive(Debug)]
-enum CguReUsable {
-    PreLto,
-    PostLto,
-    No
-}
-
 fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                  cgu: &CodegenUnit<'tcx>)
-                                 -> CguReUsable {
+                                 -> CguReuse {
     if !tcx.dep_graph.is_fully_enabled() {
-        return CguReUsable::No
+        return CguReuse::No
     }
 
     let work_product_id = &cgu.work_product_id();
     if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
         // We don't have anything cached for this CGU. This can happen
         // if the CGU did not exist in the previous session.
-        return CguReUsable::No
+        return CguReuse::No
     }
 
     // Try to mark the CGU as green. If it we can do so, it means that nothing
@@ -732,12 +726,12 @@ fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
     if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {
         // We can re-use either the pre- or the post-thinlto state
         if tcx.sess.lto() != Lto::No {
-            CguReUsable::PreLto
+            CguReuse::PreLto
         } else {
-            CguReUsable::PostLto
+            CguReuse::PostLto
         }
     } else {
-        CguReUsable::No
+        CguReuse::No
     }
 }
 
@@ -894,8 +888,11 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         ongoing_codegen.wait_for_signal_to_codegen_item();
         ongoing_codegen.check_for_errors(tcx.sess);
 
-        let loaded_from_cache = match determine_cgu_reuse(tcx, &cgu) {
-            CguReUsable::No => {
+        let cgu_reuse = determine_cgu_reuse(tcx, &cgu);
+        tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
+
+        match cgu_reuse {
+            CguReuse::No => {
                 let _timing_guard = time_graph.as_ref().map(|time_graph| {
                     time_graph.start(write::CODEGEN_WORKER_TIMELINE,
                                      write::CODEGEN_WORK_PACKAGE_KIND,
@@ -907,14 +904,14 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                 total_codegen_time += start_time.elapsed();
                 false
             }
-            CguReUsable::PreLto => {
+            CguReuse::PreLto => {
                 write::submit_pre_lto_module_to_llvm(tcx, CachedModuleCodegen {
                     name: cgu.name().to_string(),
                     source: cgu.work_product(tcx),
                 });
                 true
             }
-            CguReUsable::PostLto => {
+            CguReuse::PostLto => {
                 write::submit_post_lto_module_to_llvm(tcx, CachedModuleCodegen {
                     name: cgu.name().to_string(),
                     source: cgu.work_product(tcx),
@@ -922,12 +919,6 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                 true
             }
         };
-
-        if tcx.dep_graph.is_fully_enabled() {
-            let dep_node = cgu.codegen_dep_node(tcx);
-            let dep_node_index = tcx.dep_graph.dep_node_index_of(&dep_node);
-            tcx.dep_graph.mark_loaded_from_cache(dep_node_index, loaded_from_cache);
-        }
     }
 
     ongoing_codegen.codegen_finished(tcx);
@@ -938,9 +929,7 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                             "codegen to LLVM IR",
                             total_codegen_time);
 
-    if tcx.sess.opts.incremental.is_some() {
-        ::rustc_incremental::assert_module_sources::assert_module_sources(tcx);
-    }
+    rustc_incremental::assert_module_sources::assert_module_sources(tcx);
 
     symbol_names_test::report_symbol_names(tcx);