Auto merge of #89978 - cjgillot:qarray, r=Mark-Simulacrum

Merge the two depkind vtables

Knowledge of `DepKind`s is managed using two arrays containing flags (is_anon, eval_always, fingerprint_style), and function pointers (forcing and loading code).

This PR aims at merging the two arrays so as to reduce unneeded indirect calls and (hopefully) increase code locality.
r? `@ghost`

3 files changed, 86 insertions, 162 deletions

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index e50a8c191ad..b216d78da94 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -13,13 +13,12 @@
 extern crate rustc_macros;
 #[macro_use]
 extern crate rustc_middle;
-#[macro_use]
-extern crate tracing;
 
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::DiagnosticBuilder;
-use rustc_middle::dep_graph;
+use rustc_middle::arena::Arena;
+use rustc_middle::dep_graph::{self, DepKindStruct};
 use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
 use rustc_middle::ty::query::{Providers, QueryEngine};
 use rustc_middle::ty::{self, TyCtxt};
@@ -29,7 +28,6 @@ use rustc_span::Span;
 #[macro_use]
 mod plumbing;
 pub use plumbing::QueryCtxt;
-use plumbing::QueryStruct;
 use rustc_query_system::query::*;
 
 mod stats;
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
index 48eb488792d..86b12b3586a 100644
--- a/compiler/rustc_query_impl/src/on_disk_cache.rs
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -219,7 +219,7 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
         // Do this *before* we clone 'latest_foreign_def_path_hashes', since
         // loading existing queries may cause us to create new DepNodes, which
         // may in turn end up invoking `store_foreign_def_id_hash`
-        tcx.dep_graph.exec_cache_promotions(QueryCtxt::from_tcx(tcx));
+        tcx.dep_graph.exec_cache_promotions(tcx);
 
         *self.serialized_data.write() = None;
     }
@@ -358,23 +358,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
             Ok(())
         })
     }
-
-    fn def_path_hash_to_def_id(&self, tcx: TyCtxt<'tcx>, hash: DefPathHash) -> DefId {
-        debug!("def_path_hash_to_def_id({:?})", hash);
-
-        let stable_crate_id = hash.stable_crate_id();
-
-        // If this is a DefPathHash from the local crate, we can look up the
-        // DefId in the tcx's `Definitions`.
-        if stable_crate_id == tcx.sess.local_stable_crate_id() {
-            tcx.definitions_untracked().local_def_path_hash_to_def_id(hash).to_def_id()
-        } else {
-            // If this is a DefPathHash from an upstream crate, let the CrateStore map
-            // it to a DefId.
-            let cnum = tcx.cstore_untracked().stable_crate_id_to_crate_num(stable_crate_id);
-            tcx.cstore_untracked().def_path_hash_to_def_id(cnum, hash)
-        }
-    }
 }
 
 impl<'sess> OnDiskCache<'sess> {
@@ -764,7 +747,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
         // If we get to this point, then all of the query inputs were green,
         // which means that the definition with this hash is guaranteed to
         // still exist in the current compilation session.
-        Ok(d.tcx().on_disk_cache.as_ref().unwrap().def_path_hash_to_def_id(d.tcx(), def_path_hash))
+        Ok(d.tcx().def_path_hash_to_def_id(def_path_hash))
     }
 }
 
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 4d1e39db0ed..8c3fbb2071c 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -3,7 +3,7 @@
 //! manage the caches, and so forth.
 
 use crate::{on_disk_cache, queries, Queries};
-use rustc_middle::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex};
 use rustc_middle::ty::tls::{self, ImplicitCtxt};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::HasDepContext;
@@ -53,36 +53,6 @@ impl QueryContext for QueryCtxt<'tcx> {
         self.queries.try_collect_active_jobs(**self)
     }
 
-    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
-        let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize];
-        (cb.try_load_from_on_disk_cache)(*self, dep_node)
-    }
-
-    fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
-        debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
-
-        // We must avoid ever having to call `force_from_dep_node()` for a
-        // `DepNode::codegen_unit`:
-        // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
-        // would always end up having to evaluate the first caller of the
-        // `codegen_unit` query that *is* reconstructible. This might very well be
-        // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
-        // to re-trigger calling the `codegen_unit` query with the right key. At
-        // that point we would already have re-done all the work we are trying to
-        // avoid doing in the first place.
-        // The solution is simple: Just explicitly call the `codegen_unit` query for
-        // each CGU, right after partitioning. This way `try_mark_green` will always
-        // hit the cache instead of having to go through `force_from_dep_node`.
-        // This assertion makes sure, we actually keep applying the solution above.
-        debug_assert!(
-            dep_node.kind != DepKind::codegen_unit,
-            "calling force_from_dep_node() on DepKind::codegen_unit"
-        );
-
-        let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize];
-        (cb.force_from_dep_node)(*self, dep_node)
-    }
-
     // Interactions with on_disk_cache
     fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
         self.queries
@@ -193,60 +163,6 @@ impl<'tcx> QueryCtxt<'tcx> {
     }
 }
 
-/// This struct stores metadata about each Query.
-///
-/// Information is retrieved by indexing the `QUERIES` array using the integer value
-/// of the `DepKind`. Overall, this allows to implement `QueryContext` using this manual
-/// jump table instead of large matches.
-pub struct QueryStruct {
-    /// The red/green evaluation system will try to mark a specific DepNode in the
-    /// dependency graph as green by recursively trying to mark the dependencies of
-    /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
-    /// where we don't know if it is red or green and we therefore actually have
-    /// to recompute its value in order to find out. Since the only piece of
-    /// information that we have at that point is the `DepNode` we are trying to
-    /// re-evaluate, we need some way to re-run a query from just that. This is what
-    /// `force_from_dep_node()` implements.
-    ///
-    /// In the general case, a `DepNode` consists of a `DepKind` and an opaque
-    /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
-    /// is usually constructed by computing a stable hash of the query-key that the
-    /// `DepNode` corresponds to. Consequently, it is not in general possible to go
-    /// back from hash to query-key (since hash functions are not reversible). For
-    /// this reason `force_from_dep_node()` is expected to fail from time to time
-    /// because we just cannot find out, from the `DepNode` alone, what the
-    /// corresponding query-key is and therefore cannot re-run the query.
-    ///
-    /// The system deals with this case letting `try_mark_green` fail which forces
-    /// the root query to be re-evaluated.
-    ///
-    /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
-    /// Fortunately, we can use some contextual information that will allow us to
-    /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
-    /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
-    /// valid `DefPathHash`. Since we also always build a huge table that maps every
-    /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
-    /// everything we need to re-run the query.
-    ///
-    /// Take the `mir_promoted` query as an example. Like many other queries, it
-    /// just has a single parameter: the `DefId` of the item it will compute the
-    /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
-    /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
-    /// is actually a `DefPathHash`, and can therefore just look up the corresponding
-    /// `DefId` in `tcx.def_path_hash_to_def_id`.
-    ///
-    /// When you implement a new query, it will likely have a corresponding new
-    /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
-    /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
-    /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
-    /// add it to the "We don't have enough information to reconstruct..." group in
-    /// the match below.
-    pub(crate) force_from_dep_node: fn(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool,
-
-    /// Invoke a query to put the on-disk cached value in memory.
-    pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
-}
-
 macro_rules! handle_cycle_error {
     ([][$tcx: expr, $error:expr]) => {{
         $error.emit();
@@ -291,14 +207,14 @@ macro_rules! is_eval_always {
 }
 
 macro_rules! hash_result {
-    ([][$hcx:expr, $result:expr]) => {{
-        dep_graph::hash_result($hcx, &$result)
+    ([]) => {{
+        Some(dep_graph::hash_result)
     }};
-    ([(no_hash) $($rest:tt)*][$hcx:expr, $result:expr]) => {{
+    ([(no_hash) $($rest:tt)*]) => {{
         None
     }};
-    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
-        hash_result!([$($modifiers)*][$($args)*])
+    ([$other:tt $($modifiers:tt)*]) => {
+        hash_result!([$($modifiers)*])
     };
 }
 
@@ -378,6 +294,7 @@ macro_rules! define_queries {
             const ANON: bool = is_anon!([$($modifiers)*]);
             const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
             const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name;
+            const HASH_RESULT: Option<fn(&mut StableHashingContext<'_>, &Self::Value) -> Fingerprint> = hash_result!([$($modifiers)*]);
 
             type Cache = query_storage::$name<$tcx>;
 
@@ -406,13 +323,6 @@ macro_rules! define_queries {
                 }
             }
 
-            fn hash_result(
-                _hcx: &mut StableHashingContext<'_>,
-                _result: &Self::Value
-            ) -> Option<Fingerprint> {
-                hash_result!([$($modifiers)*][_hcx, _result])
-            }
-
             fn handle_cycle_error(
                 tcx: QueryCtxt<'tcx>,
                 mut error: DiagnosticBuilder<'_>,
@@ -421,7 +331,7 @@ macro_rules! define_queries {
             }
         })*
 
-        #[allow(non_upper_case_globals)]
+        #[allow(nonstandard_style)]
         pub mod query_callbacks {
             use super::*;
             use rustc_middle::dep_graph::DepNode;
@@ -431,68 +341,101 @@ macro_rules! define_queries {
             use rustc_query_system::dep_graph::FingerprintStyle;
 
             // We use this for most things when incr. comp. is turned off.
-            pub const Null: QueryStruct = QueryStruct {
-                force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn Null() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            pub const TraitSelect: QueryStruct = QueryStruct {
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn TraitSelect() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: true,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            pub const CompileCodegenUnit: QueryStruct = QueryStruct {
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn CompileCodegenUnit() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            pub const CompileMonoItem: QueryStruct = QueryStruct {
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn CompileMonoItem() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            $(pub const $name: QueryStruct = {
-                const is_anon: bool = is_anon!([$($modifiers)*]);
+            $(pub fn $name()-> DepKindStruct {
+                let is_anon = is_anon!([$($modifiers)*]);
+                let is_eval_always = is_eval_always!([$($modifiers)*]);
 
-                #[inline(always)]
-                fn fingerprint_style() -> FingerprintStyle {
-                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>
-                        ::fingerprint_style()
-                }
+                let fingerprint_style =
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::fingerprint_style();
 
-                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<query_keys::$name<'tcx>> {
-                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
+                if is_anon || !fingerprint_style.reconstructible() {
+                    return DepKindStruct {
+                        is_anon,
+                        is_eval_always,
+                        fingerprint_style,
+                        force_from_dep_node: None,
+                        try_load_from_on_disk_cache: None,
+                    }
                 }
 
-                fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool {
-                    force_query::<queries::$name<'_>, _>(tcx, dep_node)
+                #[inline(always)]
+                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option<query_keys::$name<'tcx>> {
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, &dep_node)
                 }
 
-                fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) {
-                    if is_anon {
-                        return
-                    }
-
-                    if !fingerprint_style().reconstructible() {
-                        return
+                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool {
+                    if let Some(key) = recover(tcx, dep_node) {
+                        let tcx = QueryCtxt::from_tcx(tcx);
+                        force_query::<queries::$name<'_>, _>(tcx, key, dep_node);
+                        true
+                    } else {
+                        false
                     }
+                }
 
-                    debug_assert!(tcx.dep_graph.is_green(dep_node));
+                fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) {
+                    debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
-                    let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+                    let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+                    let tcx = QueryCtxt::from_tcx(tcx);
                     if queries::$name::cache_on_disk(tcx, &key, None) {
                         let _ = tcx.$name(key);
                     }
                 }
 
-                QueryStruct {
-                    force_from_dep_node,
-                    try_load_from_on_disk_cache,
+                DepKindStruct {
+                    is_anon,
+                    is_eval_always,
+                    fingerprint_style,
+                    force_from_dep_node: Some(force_from_dep_node),
+                    try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
                 }
-            };)*
+            })*
         }
 
-        static QUERY_CALLBACKS: &[QueryStruct] = &make_dep_kind_array!(query_callbacks);
+        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
+            arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
+        }
     }
 }