Encode dep graph edges directly from the previous graph when promoting

4 files changed, 164 insertions, 44 deletions

diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs
index 357f2ae92d4..26aaa24771f 100644
--- a/compiler/rustc_incremental/src/persist/load.rs
+++ b/compiler/rustc_incremental/src/persist/load.rs
@@ -11,6 +11,7 @@ use rustc_session::config::IncrementalStateAssertion;
 use rustc_session::Session;
 use rustc_span::ErrorGuaranteed;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use super::data::*;
 use super::file_format;
@@ -88,7 +89,7 @@ fn delete_dirty_work_product(sess: &Session, swp: SerializedWorkProduct) {
     work_product::delete_workproduct_files(sess, &swp.work_product);
 }
 
-fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProductMap)> {
+fn load_dep_graph(sess: &Session) -> LoadResult<(Arc<SerializedDepGraph>, WorkProductMap)> {
     let prof = sess.prof.clone();
 
     if sess.opts.incremental.is_none() {
diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs
index 32759f5284a..9777f769280 100644
--- a/compiler/rustc_incremental/src/persist/save.rs
+++ b/compiler/rustc_incremental/src/persist/save.rs
@@ -10,6 +10,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_serialize::Encodable as RustcEncodable;
 use rustc_session::Session;
 use std::fs;
+use std::sync::Arc;
 
 use super::data::*;
 use super::dirty_clean;
@@ -147,7 +148,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult
 /// and moves it to the permanent dep-graph path
 pub(crate) fn build_dep_graph(
     sess: &Session,
-    prev_graph: SerializedDepGraph,
+    prev_graph: Arc<SerializedDepGraph>,
     prev_work_products: WorkProductMap,
 ) -> Option<DepGraph> {
     if sess.opts.incremental.is_none() {
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 9f067273f35..4cf8ead8ef8 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -13,6 +13,7 @@ use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
 use std::sync::atomic::Ordering;
+use std::sync::Arc;
 
 use super::query::DepGraphQuery;
 use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
@@ -81,7 +82,7 @@ pub(crate) struct DepGraphData<D: Deps> {
 
     /// The dep-graph from the previous compilation session. It contains all
     /// nodes and edges as well as all fingerprints of nodes that have them.
-    previous: SerializedDepGraph,
+    previous: Arc<SerializedDepGraph>,
 
     colors: DepNodeColorMap,
 
@@ -113,7 +114,7 @@ where
 impl<D: Deps> DepGraph<D> {
     pub fn new(
         profiler: &SelfProfilerRef,
-        prev_graph: SerializedDepGraph,
+        prev_graph: Arc<SerializedDepGraph>,
         prev_work_products: WorkProductMap,
         encoder: FileEncoder,
         record_graph: bool,
@@ -127,6 +128,7 @@ impl<D: Deps> DepGraph<D> {
             encoder,
             record_graph,
             record_stats,
+            prev_graph.clone(),
         );
 
         let colors = DepNodeColorMap::new(prev_graph_node_count);
@@ -1084,6 +1086,7 @@ impl<D: Deps> CurrentDepGraph<D> {
         encoder: FileEncoder,
         record_graph: bool,
         record_stats: bool,
+        previous: Arc<SerializedDepGraph>,
     ) -> Self {
         use std::time::{SystemTime, UNIX_EPOCH};
 
@@ -1116,6 +1119,7 @@ impl<D: Deps> CurrentDepGraph<D> {
                 record_graph,
                 record_stats,
                 profiler,
+                previous,
             ),
             new_node_to_index: Sharded::new(|| {
                 FxHashMap::with_capacity_and_hasher(
@@ -1236,16 +1240,14 @@ impl<D: Deps> CurrentDepGraph<D> {
         match prev_index_to_index[prev_index] {
             Some(dep_node_index) => dep_node_index,
             None => {
-                let key = prev_graph.index_to_node(prev_index);
-                let edges = prev_graph
-                    .edge_targets_from(prev_index)
-                    .map(|i| prev_index_to_index[i].unwrap())
-                    .collect();
-                let fingerprint = prev_graph.fingerprint_by_index(prev_index);
-                let dep_node_index = self.encoder.send(key, fingerprint, edges);
+                let dep_node_index = self.encoder.promote(prev_index, &*prev_index_to_index);
                 prev_index_to_index[prev_index] = Some(dep_node_index);
                 #[cfg(debug_assertions)]
-                self.record_edge(dep_node_index, key, fingerprint);
+                self.record_edge(
+                    dep_node_index,
+                    prev_graph.index_to_node(prev_index),
+                    prev_graph.fingerprint_by_index(prev_index),
+                );
                 dep_node_index
             }
         }
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index 0c6a6358293..b5c610f817d 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -41,6 +41,7 @@ use crate::dep_graph::edges::EdgesVec;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fingerprint::PackedFingerprint;
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::outline;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::unhash::UnhashMap;
@@ -49,6 +50,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixed
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use std::iter;
 use std::marker::PhantomData;
+use std::sync::Arc;
 
 // The maximum value of `SerializedDepNodeIndex` leaves the upper two bits
 // unused so that we can store multiple index types in `CompressedHybridIndex`,
@@ -94,7 +96,7 @@ impl SerializedDepGraph {
     pub fn edge_targets_from(
         &self,
         source: SerializedDepNodeIndex,
-    ) -> impl Iterator<Item = SerializedDepNodeIndex> + '_ {
+    ) -> impl Iterator<Item = SerializedDepNodeIndex> + Clone + '_ {
         let header = self.edge_list_indices[source];
         let mut raw = &self.edge_list_data[header.start()..];
         // Figure out where the edge list for `source` ends by getting the start index of the next
@@ -176,7 +178,7 @@ fn mask(bits: usize) -> usize {
 
 impl SerializedDepGraph {
     #[instrument(level = "debug", skip(d))]
-    pub fn decode<D: Deps>(d: &mut MemDecoder<'_>) -> SerializedDepGraph {
+    pub fn decode<D: Deps>(d: &mut MemDecoder<'_>) -> Arc<SerializedDepGraph> {
         // The last 16 bytes are the node count and edge count.
         debug!("position: {:?}", d.position());
         let (node_count, edge_count, graph_size) =
@@ -254,7 +256,13 @@ impl SerializedDepGraph {
             index[node.kind.as_usize()].insert(node.hash, idx);
         }
 
-        SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index }
+        Arc::new(SerializedDepGraph {
+            nodes,
+            fingerprints,
+            edge_list_indices,
+            edge_list_data,
+            index,
+        })
     }
 }
 
@@ -299,21 +307,24 @@ impl<D: Deps> SerializedNodeHeader<D> {
     const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1;
 
     #[inline]
-    fn new(node_info: &NodeInfo) -> Self {
+    fn new(
+        node: DepNode,
+        fingerprint: Fingerprint,
+        edge_max_index: u32,
+        edge_count: usize,
+    ) -> Self {
         debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS);
 
-        let NodeInfo { node, fingerprint, edges } = node_info;
-
         let mut head = node.kind.as_inner();
 
-        let free_bytes = edges.max_index().leading_zeros() as usize / 8;
+        let free_bytes = edge_max_index.leading_zeros() as usize / 8;
         let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1);
         head |= (bytes_per_index as u16) << Self::KIND_BITS;
 
         // Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit
         // in this bitfield.
-        if edges.len() <= Self::MAX_INLINE_LEN {
-            head |= (edges.len() as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
+        if edge_count <= Self::MAX_INLINE_LEN {
+            head |= (edge_count as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
         }
 
         let hash: Fingerprint = node.hash.into();
@@ -327,10 +338,10 @@ impl<D: Deps> SerializedNodeHeader<D> {
         #[cfg(debug_assertions)]
         {
             let res = Self { bytes, _marker: PhantomData };
-            assert_eq!(node_info.fingerprint, res.fingerprint());
-            assert_eq!(node_info.node, res.node());
+            assert_eq!(fingerprint, res.fingerprint());
+            assert_eq!(node, res.node());
             if let Some(len) = res.len() {
-                assert_eq!(node_info.edges.len(), len);
+                assert_eq!(edge_count, len);
             }
         }
         Self { bytes, _marker: PhantomData }
@@ -393,20 +404,55 @@ struct NodeInfo {
 
 impl NodeInfo {
     fn encode<D: Deps>(&self, e: &mut FileEncoder) {
-        let header = SerializedNodeHeader::<D>::new(self);
+        let NodeInfo { node, fingerprint, ref edges } = *self;
+        let header =
+            SerializedNodeHeader::<D>::new(node, fingerprint, edges.max_index(), edges.len());
+        e.write_array(header.bytes);
+
+        if header.len().is_none() {
+            e.emit_usize(edges.len());
+        }
+
+        let bytes_per_index = header.bytes_per_index();
+        for node_index in edges.iter() {
+            e.write_with(|dest| {
+                *dest = node_index.as_u32().to_le_bytes();
+                bytes_per_index
+            });
+        }
+    }
+
+    #[inline]
+    fn encode_promoted<D: Deps>(
+        e: &mut FileEncoder,
+        node: DepNode,
+        fingerprint: Fingerprint,
+        prev_index: SerializedDepNodeIndex,
+        prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
+        previous: &SerializedDepGraph,
+    ) -> usize {
+        let edges = previous.edge_targets_from(prev_index);
+        let edge_count = edges.size_hint().0;
+        let edge_max =
+            edges.clone().map(|i| prev_index_to_index[i].unwrap().as_u32()).max().unwrap_or(0);
+
+        let header = SerializedNodeHeader::<D>::new(node, fingerprint, edge_max, edge_count);
         e.write_array(header.bytes);
 
         if header.len().is_none() {
-            e.emit_usize(self.edges.len());
+            e.emit_usize(edge_count);
         }
 
         let bytes_per_index = header.bytes_per_index();
-        for node_index in self.edges.iter() {
+        for node_index in edges {
+            let node_index = prev_index_to_index[node_index].unwrap();
             e.write_with(|dest| {
                 *dest = node_index.as_u32().to_le_bytes();
                 bytes_per_index
             });
         }
+
+        edge_count
     }
 }
 
@@ -417,6 +463,7 @@ struct Stat {
 }
 
 struct EncoderState<D: Deps> {
+    previous: Arc<SerializedDepGraph>,
     encoder: FileEncoder,
     total_node_count: usize,
     total_edge_count: usize,
@@ -428,8 +475,9 @@ struct EncoderState<D: Deps> {
 }
 
 impl<D: Deps> EncoderState<D> {
-    fn new(encoder: FileEncoder, record_stats: bool) -> Self {
+    fn new(encoder: FileEncoder, record_stats: bool, previous: Arc<SerializedDepGraph>) -> Self {
         Self {
+            previous,
             encoder,
             total_edge_count: 0,
             total_node_count: 0,
@@ -439,38 +487,90 @@ impl<D: Deps> EncoderState<D> {
         }
     }
 
-    fn encode_node(
+    #[inline]
+    fn record(
         &mut self,
-        node: &NodeInfo,
+        node: DepNode,
+        edge_count: usize,
+        edges: impl FnOnce(&mut Self) -> Vec<DepNodeIndex>,
         record_graph: &Option<Lock<DepGraphQuery>>,
     ) -> DepNodeIndex {
         let index = DepNodeIndex::new(self.total_node_count);
-        self.total_node_count += 1;
-        self.kind_stats[node.node.kind.as_usize()] += 1;
 
-        let edge_count = node.edges.len();
+        self.total_node_count += 1;
+        self.kind_stats[node.kind.as_usize()] += 1;
         self.total_edge_count += edge_count;
 
         if let Some(record_graph) = &record_graph {
-            // Do not ICE when a query is called from within `with_query`.
-            if let Some(record_graph) = &mut record_graph.try_lock() {
-                record_graph.push(index, node.node, &node.edges);
-            }
+            let edges = edges(self);
+            outline(move || {
+                // Do not ICE when a query is called from within `with_query`.
+                if let Some(record_graph) = &mut record_graph.try_lock() {
+                    record_graph.push(index, node, &edges);
+                }
+            });
         }
 
         if let Some(stats) = &mut self.stats {
-            let kind = node.node.kind;
+            let kind = node.kind;
 
-            let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
-            stat.node_counter += 1;
-            stat.edge_counter += edge_count as u64;
+            outline(move || {
+                let stat =
+                    stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
+                stat.node_counter += 1;
+                stat.edge_counter += edge_count as u64;
+            });
         }
 
-        let encoder = &mut self.encoder;
-        node.encode::<D>(encoder);
         index
     }
 
+    fn encode_node(
+        &mut self,
+        node: &NodeInfo,
+        record_graph: &Option<Lock<DepGraphQuery>>,
+    ) -> DepNodeIndex {
+        node.encode::<D>(&mut self.encoder);
+        self.record(
+            node.node,
+            node.edges.len(),
+            |_| node.edges[..].iter().copied().collect(),
+            record_graph,
+        )
+    }
+
+    #[inline]
+    fn promote_node(
+        &mut self,
+        prev_index: SerializedDepNodeIndex,
+        record_graph: &Option<Lock<DepGraphQuery>>,
+        prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
+    ) -> DepNodeIndex {
+        let node = self.previous.index_to_node(prev_index);
+
+        let fingerprint = self.previous.fingerprint_by_index(prev_index);
+        let edge_count = NodeInfo::encode_promoted::<D>(
+            &mut self.encoder,
+            node,
+            fingerprint,
+            prev_index,
+            prev_index_to_index,
+            &self.previous,
+        );
+
+        self.record(
+            node,
+            edge_count,
+            |this| {
+                this.previous
+                    .edge_targets_from(prev_index)
+                    .map(|i| prev_index_to_index[i].unwrap())
+                    .collect()
+            },
+            record_graph,
+        )
+    }
+
     fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let Self {
             mut encoder,
@@ -479,6 +579,7 @@ impl<D: Deps> EncoderState<D> {
             stats: _,
             kind_stats,
             marker: _,
+            previous: _,
         } = self;
 
         let node_count = total_node_count.try_into().unwrap();
@@ -520,9 +621,10 @@ impl<D: Deps> GraphEncoder<D> {
         record_graph: bool,
         record_stats: bool,
         profiler: &SelfProfilerRef,
+        previous: Arc<SerializedDepGraph>,
     ) -> Self {
         let record_graph = record_graph.then(|| Lock::new(DepGraphQuery::new(prev_node_count)));
-        let status = Lock::new(Some(EncoderState::new(encoder, record_stats)));
+        let status = Lock::new(Some(EncoderState::new(encoder, record_stats, previous)));
         GraphEncoder { status, record_graph, profiler: profiler.clone() }
     }
 
@@ -596,6 +698,20 @@ impl<D: Deps> GraphEncoder<D> {
         self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph)
     }
 
+    #[inline]
+    pub(crate) fn promote(
+        &self,
+        prev_index: SerializedDepNodeIndex,
+        prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
+    ) -> DepNodeIndex {
+        let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph");
+        self.status.lock().as_mut().unwrap().promote_node(
+            prev_index,
+            &self.record_graph,
+            prev_index_to_index,
+        )
+    }
+
     pub fn finish(&self) -> FileEncodeResult {
         let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish");