Auto merge of #39109 - michaelwoerister:incr-comp-cache-cleanup, r=nikomatsakis

incr.comp.: Delete orphaned work-products.

The new partitioning scheme uncovered a hole in our incr. comp. cache directory garbage collection. So far, we relied on unneeded work products being deleted during the initial cache invalidation phase. However, we the new scheme, we get object files/work products that only contain code from upstream crates. Sometimes this code is not needed anymore (because all callers have been removed from the source) but because nothing that actually influences the contents of these work products had changed, we never deleted them from disk.

r? @nikomatsakis

6 files changed, 51 insertions, 12 deletions

diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs
index 26e1dc7e049..e6736ccafba 100644
--- a/src/librustc/dep_graph/graph.rs
+++ b/src/librustc/dep_graph/graph.rs
@@ -126,6 +126,12 @@ impl DepGraph {
     pub fn work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
         self.data.work_products.borrow()
     }
+
+    /// Access the map of work-products created during the cached run. Only
+    /// used during saving of the dep-graph.
+    pub fn previous_work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
+        self.data.previous_work_products.borrow()
+    }
 }
 
 /// A "work product" is an intermediate result that we save into the
diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs
index ea0a0034c3e..e96d5977419 100644
--- a/src/librustc_incremental/lib.rs
+++ b/src/librustc_incremental/lib.rs
@@ -57,3 +57,4 @@ pub use persist::save_trans_partition;
 pub use persist::save_work_products;
 pub use persist::in_incr_comp_dir;
 pub use persist::finalize_session_directory;
+pub use persist::delete_workproduct_files;
diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs
index 8ff04a565e9..433110a2a6d 100644
--- a/src/librustc_incremental/persist/load.rs
+++ b/src/librustc_incremental/persist/load.rs
@@ -18,7 +18,6 @@ use rustc::ty::TyCtxt;
 use rustc_data_structures::fx::{FxHashSet, FxHashMap};
 use rustc_serialize::Decodable as RustcDecodable;
 use rustc_serialize::opaque::Decoder;
-use std::fs;
 use std::path::{Path};
 
 use IncrementalHashesMap;
@@ -29,6 +28,7 @@ use super::dirty_clean;
 use super::hash::*;
 use super::fs::*;
 use super::file_format;
+use super::work_product;
 
 pub type DirtyNodes = FxHashSet<DepNode<DefPathIndex>>;
 
@@ -322,17 +322,7 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 fn delete_dirty_work_product(tcx: TyCtxt,
                              swp: SerializedWorkProduct) {
     debug!("delete_dirty_work_product({:?})", swp);
-    for &(_, ref file_name) in &swp.work_product.saved_files {
-        let path = in_incr_comp_dir_sess(tcx.sess, file_name);
-        match fs::remove_file(&path) {
-            Ok(()) => { }
-            Err(err) => {
-                tcx.sess.warn(
-                    &format!("file-system error deleting outdated file `{}`: {}",
-                             path.display(), err));
-            }
-        }
-    }
+    work_product::delete_workproduct_files(tcx.sess, &swp.work_product);
 }
 
 fn load_prev_metadata_hashes(tcx: TyCtxt,
diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs
index 26fcde05868..3ae8fcdfb51 100644
--- a/src/librustc_incremental/persist/mod.rs
+++ b/src/librustc_incremental/persist/mod.rs
@@ -29,3 +29,4 @@ pub use self::load::load_dep_graph;
 pub use self::save::save_dep_graph;
 pub use self::save::save_work_products;
 pub use self::work_product::save_trans_partition;
+pub use self::work_product::delete_workproduct_files;
diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs
index 14f1e222755..356eb845fed 100644
--- a/src/librustc_incremental/persist/save.rs
+++ b/src/librustc_incremental/persist/save.rs
@@ -30,6 +30,7 @@ use super::preds::*;
 use super::fs::*;
 use super::dirty_clean;
 use super::file_format;
+use super::work_product;
 use calculate_svh::IchHasher;
 
 pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -84,6 +85,31 @@ pub fn save_work_products(sess: &Session) {
     let _ignore = sess.dep_graph.in_ignore();
     let path = work_products_path(sess);
     save_in(sess, path, |e| encode_work_products(sess, e));
+
+    // We also need to clean out old work-products, as not all of them are
+    // deleted during invalidation. Some object files don't change their
+    // content, they are just not needed anymore.
+    let new_work_products = sess.dep_graph.work_products();
+    let previous_work_products = sess.dep_graph.previous_work_products();
+
+    for (id, wp) in previous_work_products.iter() {
+        if !new_work_products.contains_key(id) {
+            work_product::delete_workproduct_files(sess, wp);
+            debug_assert!(wp.saved_files.iter().all(|&(_, ref file_name)| {
+                !in_incr_comp_dir_sess(sess, file_name).exists()
+            }));
+        }
+    }
+
+    // Check that we did not delete one of the current work-products:
+    debug_assert!({
+        new_work_products.iter()
+                         .flat_map(|(_, wp)| wp.saved_files
+                                               .iter()
+                                               .map(|&(_, ref name)| name))
+                         .map(|name| in_incr_comp_dir_sess(sess, name))
+                         .all(|path| path.exists())
+    });
 }
 
 fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs
index a9ebd27ce99..0e9f9061540 100644
--- a/src/librustc_incremental/persist/work_product.rs
+++ b/src/librustc_incremental/persist/work_product.rs
@@ -17,6 +17,7 @@ use rustc::session::config::OutputType;
 use rustc::util::fs::link_or_copy;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::fs as std_fs;
 
 pub fn save_trans_partition(sess: &Session,
                             cgu_name: &str,
@@ -61,3 +62,17 @@ pub fn save_trans_partition(sess: &Session,
 
     sess.dep_graph.insert_work_product(&work_product_id, work_product);
 }
+
+pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) {
+    for &(_, ref file_name) in &work_product.saved_files {
+        let path = in_incr_comp_dir_sess(sess, file_name);
+        match std_fs::remove_file(&path) {
+            Ok(()) => { }
+            Err(err) => {
+                sess.warn(
+                    &format!("file-system error deleting outdated file `{}`: {}",
+                             path.display(), err));
+            }
+        }
+    }
+}