Merge from rustc

2 files changed, 114 insertions, 52 deletions

diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 9f34417973e..0d56db16099 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -141,7 +141,7 @@ impl<D: Deps> DepGraph<D> {
         let colors = DepNodeColorMap::new(prev_graph_node_count);
 
         // Instantiate a node with zero dependencies only once for anonymous queries.
-        let _green_node_index = current.alloc_node(
+        let _green_node_index = current.alloc_new_node(
             DepNode { kind: D::DEP_KIND_ANON_ZERO_DEPS, hash: current.anon_id_seed.into() },
             EdgesVec::new(),
             Fingerprint::ZERO,
@@ -149,7 +149,7 @@ impl<D: Deps> DepGraph<D> {
         assert_eq!(_green_node_index, DepNodeIndex::SINGLETON_ZERO_DEPS_ANON_NODE);
 
         // Instantiate a dependy-less red node only once for anonymous queries.
-        let red_node_index = current.alloc_node(
+        let red_node_index = current.alloc_new_node(
             DepNode { kind: D::DEP_KIND_RED, hash: Fingerprint::ZERO.into() },
             EdgesVec::new(),
             Fingerprint::ZERO,
@@ -438,7 +438,7 @@ impl<D: Deps> DepGraphData<D> {
                 // memory impact of this `anon_node_to_index` map remains tolerable, and helps
                 // us avoid useless growth of the graph with almost-equivalent nodes.
                 self.current.anon_node_to_index.get_or_insert_with(target_dep_node, || {
-                    self.current.alloc_node(target_dep_node, task_deps, Fingerprint::ZERO)
+                    self.current.alloc_new_node(target_dep_node, task_deps, Fingerprint::ZERO)
                 })
             }
         };
@@ -680,8 +680,8 @@ impl<D: Deps> DepGraphData<D> {
         qcx: Qcx,
         diagnostic: &DiagInner,
     ) -> DepNodeIndex {
-        // Use `send` so we get an unique index, even though the dep node is not.
-        let dep_node_index = self.current.encoder.send(
+        // Use `send_new` so we get an unique index, even though the dep node is not.
+        let dep_node_index = self.current.encoder.send_new(
             DepNode {
                 kind: D::DEP_KIND_SIDE_EFFECT,
                 hash: PackedFingerprint::from(Fingerprint::ZERO),
@@ -713,20 +713,22 @@ impl<D: Deps> DepGraphData<D> {
                 }
             }
 
-            // Manually recreate the node as `promote_node_and_deps_to_current` expects all
-            // green dependencies.
-            let dep_node_index = self.current.encoder.send(
+            // Use `send_and_color` as `promote_node_and_deps_to_current` expects all
+            // green dependencies. `send_and_color` will also prevent multiple nodes
+            // being encoded for concurrent calls.
+            let dep_node_index = self.current.encoder.send_and_color(
+                prev_index,
+                &self.colors,
                 DepNode {
                     kind: D::DEP_KIND_SIDE_EFFECT,
                     hash: PackedFingerprint::from(Fingerprint::ZERO),
                 },
                 Fingerprint::ZERO,
                 std::iter::once(DepNodeIndex::FOREVER_RED_NODE).collect(),
+                true,
             );
+            // This will just overwrite the same value for concurrent calls.
             qcx.store_side_effect(dep_node_index, side_effect);
-
-            // Mark the node as green.
-            self.colors.insert(prev_index, DepNodeColor::Green(dep_node_index));
         })
     }
 
@@ -736,38 +738,43 @@ impl<D: Deps> DepGraphData<D> {
         edges: EdgesVec,
         fingerprint: Option<Fingerprint>,
     ) -> DepNodeIndex {
-        let dep_node_index =
-            self.current.alloc_node(key, edges, fingerprint.unwrap_or(Fingerprint::ZERO));
-
         if let Some(prev_index) = self.previous.node_to_index_opt(&key) {
             // Determine the color and index of the new `DepNode`.
-            let color = if let Some(fingerprint) = fingerprint {
+            let is_green = if let Some(fingerprint) = fingerprint {
                 if fingerprint == self.previous.fingerprint_by_index(prev_index) {
                     // This is a green node: it existed in the previous compilation,
                     // its query was re-executed, and it has the same result as before.
-                    DepNodeColor::Green(dep_node_index)
+                    true
                 } else {
                     // This is a red node: it existed in the previous compilation, its query
                     // was re-executed, but it has a different result from before.
-                    DepNodeColor::Red
+                    false
                 }
             } else {
                 // This is a red node, effectively: it existed in the previous compilation
                 // session, its query was re-executed, but it doesn't compute a result hash
                 // (i.e. it represents a `no_hash` query), so we have no way of determining
                 // whether or not the result was the same as before.
-                DepNodeColor::Red
+                false
             };
 
-            debug_assert!(
-                self.colors.get(prev_index).is_none(),
-                "DepGraph::with_task() - Duplicate DepNodeColor insertion for {key:?}",
+            let fingerprint = fingerprint.unwrap_or(Fingerprint::ZERO);
+
+            let dep_node_index = self.current.encoder.send_and_color(
+                prev_index,
+                &self.colors,
+                key,
+                fingerprint,
+                edges,
+                is_green,
             );
 
-            self.colors.insert(prev_index, color);
-        }
+            self.current.record_node(dep_node_index, key, fingerprint);
 
-        dep_node_index
+            dep_node_index
+        } else {
+            self.current.alloc_new_node(key, edges, fingerprint.unwrap_or(Fingerprint::ZERO))
+        }
     }
 
     fn promote_node_and_deps_to_current(&self, prev_index: SerializedDepNodeIndex) -> DepNodeIndex {
@@ -1167,8 +1174,7 @@ pub(super) struct CurrentDepGraph<D: Deps> {
     /// ID from the previous session. In order to side-step this problem, we make
     /// sure that anonymous `NodeId`s allocated in different sessions don't overlap.
     /// This is implemented by mixing a session-key into the ID fingerprint of
-    /// each anon node. The session-key is just a random number generated when
-    /// the `DepGraph` is created.
+    /// each anon node. The session-key is a hash of the number of previous sessions.
     anon_id_seed: Fingerprint,
 
     /// These are simple counters that are for profiling and
@@ -1186,12 +1192,8 @@ impl<D: Deps> CurrentDepGraph<D> {
         record_stats: bool,
         previous: Arc<SerializedDepGraph>,
     ) -> Self {
-        use std::time::{SystemTime, UNIX_EPOCH};
-
-        let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
-        let nanos = duration.as_nanos();
         let mut stable_hasher = StableHasher::new();
-        nanos.hash(&mut stable_hasher);
+        previous.session_count().hash(&mut stable_hasher);
         let anon_id_seed = stable_hasher.finish();
 
         #[cfg(debug_assertions)]
@@ -1246,19 +1248,15 @@ impl<D: Deps> CurrentDepGraph<D> {
         assert_eq!(previous, fingerprint, "Unstable fingerprints for {:?}", key);
     }
 
-    /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
-    /// Assumes that this is a node that has no equivalent in the previous dep-graph.
     #[inline(always)]
-    fn alloc_node(
+    fn record_node(
         &self,
+        dep_node_index: DepNodeIndex,
         key: DepNode,
-        edges: EdgesVec,
-        current_fingerprint: Fingerprint,
-    ) -> DepNodeIndex {
-        let dep_node_index = self.encoder.send(key, current_fingerprint, edges);
-
+        _current_fingerprint: Fingerprint,
+    ) {
         #[cfg(debug_assertions)]
-        self.record_edge(dep_node_index, key, current_fingerprint);
+        self.record_edge(dep_node_index, key, _current_fingerprint);
 
         if let Some(ref nodes_in_current_session) = self.nodes_in_current_session {
             outline(|| {
@@ -1267,6 +1265,20 @@ impl<D: Deps> CurrentDepGraph<D> {
                 }
             });
         }
+    }
+
+    /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
+    /// Assumes that this is a node that has no equivalent in the previous dep-graph.
+    #[inline(always)]
+    fn alloc_new_node(
+        &self,
+        key: DepNode,
+        edges: EdgesVec,
+        current_fingerprint: Fingerprint,
+    ) -> DepNodeIndex {
+        let dep_node_index = self.encoder.send_new(key, current_fingerprint, edges);
+
+        self.record_node(dep_node_index, key, current_fingerprint);
 
         dep_node_index
     }
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index 7750d6d1fef..ac24628447d 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -92,6 +92,9 @@ pub struct SerializedDepGraph {
     /// Stores a map from fingerprints to nodes per dep node kind.
     /// This is the reciprocal of `nodes`.
     index: Vec<UnhashMap<PackedFingerprint, SerializedDepNodeIndex>>,
+    /// The number of previous compilation sessions. This is used to generate
+    /// unique anon dep nodes per session.
+    session_count: u64,
 }
 
 impl SerializedDepGraph {
@@ -146,6 +149,11 @@ impl SerializedDepGraph {
     pub fn node_count(&self) -> usize {
         self.nodes.len()
     }
+
+    #[inline]
+    pub fn session_count(&self) -> u64 {
+        self.session_count
+    }
 }
 
 /// A packed representation of an edge's start index and byte width.
@@ -226,12 +234,12 @@ impl SerializedDepGraph {
 
             // If the length of this node's edge list is small, the length is stored in the header.
             // If it is not, we fall back to another decoder call.
-            let num_edges = node_header.len().unwrap_or_else(|| d.read_usize());
+            let num_edges = node_header.len().unwrap_or_else(|| d.read_u32());
 
             // The edges index list uses the same varint strategy as rmeta tables; we select the
             // number of byte elements per-array not per-element. This lets us read the whole edge
             // list for a node with one decoder call and also use the on-disk format in memory.
-            let edges_len_bytes = node_header.bytes_per_index() * num_edges;
+            let edges_len_bytes = node_header.bytes_per_index() * (num_edges as usize);
             // The in-memory structure for the edges list stores the byte width of the edges on
             // this node with the offset into the global edge data array.
             let edges_header = node_header.edges_header(&edge_list_data);
@@ -252,6 +260,8 @@ impl SerializedDepGraph {
             .map(|_| UnhashMap::with_capacity_and_hasher(d.read_u32() as usize, Default::default()))
             .collect();
 
+        let session_count = d.read_u64();
+
         for (idx, node) in nodes.iter_enumerated() {
             if index[node.kind.as_usize()].insert(node.hash, idx).is_some() {
                 // Side effect nodes can have duplicates
@@ -273,6 +283,7 @@ impl SerializedDepGraph {
             edge_list_indices,
             edge_list_data,
             index,
+            session_count,
         })
     }
 }
@@ -296,7 +307,7 @@ struct SerializedNodeHeader<D> {
 // The fields of a `SerializedNodeHeader`, this struct is an implementation detail and exists only
 // to make the implementation of `SerializedNodeHeader` simpler.
 struct Unpacked {
-    len: Option<usize>,
+    len: Option<u32>,
     bytes_per_index: usize,
     kind: DepKind,
     hash: PackedFingerprint,
@@ -352,7 +363,7 @@ impl<D: Deps> SerializedNodeHeader<D> {
             assert_eq!(fingerprint, res.fingerprint());
             assert_eq!(node, res.node());
             if let Some(len) = res.len() {
-                assert_eq!(edge_count, len);
+                assert_eq!(edge_count, len as usize);
             }
         }
         Self { bytes, _marker: PhantomData }
@@ -366,7 +377,7 @@ impl<D: Deps> SerializedNodeHeader<D> {
 
         let kind = head & mask(Self::KIND_BITS) as u16;
         let bytes_per_index = (head >> Self::KIND_BITS) & mask(Self::WIDTH_BITS) as u16;
-        let len = (head as usize) >> (Self::WIDTH_BITS + Self::KIND_BITS);
+        let len = (head as u32) >> (Self::WIDTH_BITS + Self::KIND_BITS);
 
         Unpacked {
             len: len.checked_sub(1),
@@ -378,7 +389,7 @@ impl<D: Deps> SerializedNodeHeader<D> {
     }
 
     #[inline]
-    fn len(&self) -> Option<usize> {
+    fn len(&self) -> Option<u32> {
         self.unpack().len
     }
 
@@ -421,7 +432,8 @@ impl NodeInfo {
         e.write_array(header.bytes);
 
         if header.len().is_none() {
-            e.emit_usize(edges.len());
+            // The edges are all unique and the number of unique indices is less than u32::MAX.
+            e.emit_u32(edges.len().try_into().unwrap());
         }
 
         let bytes_per_index = header.bytes_per_index();
@@ -456,7 +468,8 @@ impl NodeInfo {
         e.write_array(header.bytes);
 
         if header.len().is_none() {
-            e.emit_usize(edge_count);
+            // The edges are all unique and the number of unique indices is less than u32::MAX.
+            e.emit_u32(edge_count.try_into().unwrap());
         }
 
         let bytes_per_index = header.bytes_per_index();
@@ -601,7 +614,7 @@ impl<D: Deps> EncoderState<D> {
             stats: _,
             kind_stats,
             marker: _,
-            previous: _,
+            previous,
         } = self;
 
         let node_count = total_node_count.try_into().unwrap();
@@ -612,6 +625,8 @@ impl<D: Deps> EncoderState<D> {
             count.encode(&mut encoder);
         }
 
+        previous.session_count.checked_add(1).unwrap().encode(&mut encoder);
+
         debug!(?node_count, ?edge_count);
         debug!("position: {:?}", encoder.position());
         IntEncodedWithFixedSize(node_count).encode(&mut encoder);
@@ -707,7 +722,8 @@ impl<D: Deps> GraphEncoder<D> {
         }
     }
 
-    pub(crate) fn send(
+    /// Encodes a node that does not exists in the previous graph.
+    pub(crate) fn send_new(
         &self,
         node: DepNode,
         fingerprint: Fingerprint,
@@ -718,6 +734,40 @@ impl<D: Deps> GraphEncoder<D> {
         self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph)
     }
 
+    /// Encodes a node that exists in the previous graph, but was re-executed.
+    ///
+    /// This will also ensure the dep node is colored either red or green.
+    pub(crate) fn send_and_color(
+        &self,
+        prev_index: SerializedDepNodeIndex,
+        colors: &DepNodeColorMap,
+        node: DepNode,
+        fingerprint: Fingerprint,
+        edges: EdgesVec,
+        is_green: bool,
+    ) -> DepNodeIndex {
+        let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph");
+        let node = NodeInfo { node, fingerprint, edges };
+
+        let mut status = self.status.lock();
+        let status = status.as_mut().unwrap();
+
+        // Check colors inside the lock to avoid racing when `send_promoted` is called concurrently
+        // on the same index.
+        match colors.get(prev_index) {
+            None => {
+                let dep_node_index = status.encode_node(&node, &self.record_graph);
+                colors.insert(
+                    prev_index,
+                    if is_green { DepNodeColor::Green(dep_node_index) } else { DepNodeColor::Red },
+                );
+                dep_node_index
+            }
+            Some(DepNodeColor::Green(dep_node_index)) => dep_node_index,
+            Some(DepNodeColor::Red) => panic!(),
+        }
+    }
+
     /// Encodes a node that was promoted from the previous graph. It reads the information directly from
     /// the previous dep graph and expects all edges to already have a new dep node index assigned.
     ///
@@ -733,8 +783,8 @@ impl<D: Deps> GraphEncoder<D> {
         let mut status = self.status.lock();
         let status = status.as_mut().unwrap();
 
-        // Check colors inside the lock to avoid racing when `send_promoted` is called concurrently
-        // on the same index.
+        // Check colors inside the lock to avoid racing when `send_promoted` or `send_and_color`
+        // is called concurrently on the same index.
         match colors.get(prev_index) {
             None => {
                 let dep_node_index =