Build jump table at runtime.

10 files changed, 111 insertions, 130 deletions

diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 8bd6ed98973..eea32083568 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -838,7 +838,7 @@ pub fn create_global_ctxt<'tcx>(
                 dep_graph,
                 queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
                 queries.as_dyn(),
-                &rustc_query_impl::QUERY_CALLBACKS,
+                rustc_query_impl::query_callbacks(arena),
                 crate_name,
                 outputs,
             )
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index 4a027cb7ebe..420c500a7de 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -100,6 +100,8 @@ macro_rules! arena_types {
             // This is used to decode the &'tcx [Span] for InlineAsm's line_spans.
             [decode] span: rustc_span::Span,
             [decode] used_trait_imports: rustc_data_structures::fx::FxHashSet<rustc_hir::def_id::LocalDefId>,
+
+            [] dep_kind: rustc_middle::dep_graph::DepKindStruct,
         ], $tcx);
     )
 }
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index ee0b4ec7a60..cd7bd56182d 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -87,7 +87,7 @@ pub struct DepKindStruct {
 
     /// Whether the query key can be recovered from the hashed fingerprint.
     /// See [DepNodeParams] trait for the behaviour of each key type.
-    pub fingerprint_style: fn() -> FingerprintStyle,
+    pub fingerprint_style: FingerprintStyle,
 
     /// The red/green evaluation system will try to mark a specific DepNode in the
     /// dependency graph as green by recursively trying to mark the dependencies of
@@ -131,10 +131,10 @@ pub struct DepKindStruct {
     /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
     /// add it to the "We don't have enough information to reconstruct..." group in
     /// the match below.
-    pub force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
+    pub force_from_dep_node: Option<fn(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool>,
 
     /// Invoke a query to put the on-disk cached value in memory.
-    pub try_load_from_on_disk_cache: fn(TyCtxt<'_>, &DepNode),
+    pub try_load_from_on_disk_cache: Option<fn(TyCtxt<'_>, DepNode)>,
 }
 
 impl DepKind {
@@ -145,8 +145,7 @@ impl DepKind {
         if data.is_anon {
             return FingerprintStyle::Opaque;
         }
-
-        (data.fingerprint_style)()
+        data.fingerprint_style
     }
 }
 
@@ -159,7 +158,7 @@ macro_rules! define_dep_nodes {
     ) => (
         #[macro_export]
         macro_rules! make_dep_kind_array {
-            ($mod:ident) => {[ $(($mod::$variant),)* ]};
+            ($mod:ident) => {[ $($mod::$variant()),* ]};
         }
 
         /// This enum serves as an index into arrays built by `make_dep_kind_array`.
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 8ac3cef9040..79d7ca32f35 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -100,7 +100,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         self.query_kind(kind).is_eval_always
     }
 
-    fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
+    fn try_force_from_dep_node(&self, dep_node: DepNode) -> bool {
         debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
 
         // We must avoid ever having to call `force_from_dep_node()` for a
@@ -122,11 +122,18 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         );
 
         let cb = self.query_kind(dep_node.kind);
-        (cb.force_from_dep_node)(*self, dep_node)
+        if let Some(f) = cb.force_from_dep_node {
+            f(*self, dep_node);
+            true
+        } else {
+            false
+        }
     }
 
-    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
+    fn try_load_from_on_disk_cache(&self, dep_node: DepNode) {
         let cb = self.query_kind(dep_node.kind);
-        (cb.try_load_from_on_disk_cache)(*self, dep_node)
+        if let Some(f) = cb.try_load_from_on_disk_cache {
+            f(*self, dep_node)
+        }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index b00a2c65575..8240273acad 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -1186,7 +1186,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
+    crate fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
         &self.query_kinds[k as usize]
     }
 
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index d86d1bb8af4..b216d78da94 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -17,6 +17,7 @@ extern crate rustc_middle;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::DiagnosticBuilder;
+use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::{self, DepKindStruct};
 use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
 use rustc_middle::ty::query::{Providers, QueryEngine};
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index be704b75b2b..8c3fbb2071c 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -331,7 +331,7 @@ macro_rules! define_queries {
             }
         })*
 
-        #[allow(non_upper_case_globals)]
+        #[allow(nonstandard_style)]
         pub mod query_callbacks {
             use super::*;
             use rustc_middle::dep_graph::DepNode;
@@ -341,74 +341,83 @@ macro_rules! define_queries {
             use rustc_query_system::dep_graph::FingerprintStyle;
 
             // We use this for most things when incr. comp. is turned off.
-            pub const Null: DepKindStruct = DepKindStruct {
-                is_anon: false,
-                is_eval_always: false,
-
-                fingerprint_style: || FingerprintStyle::Unit,
-                force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
-                try_load_from_on_disk_cache: |_, _| {},
-            };
-
-            pub const TraitSelect: DepKindStruct = DepKindStruct {
-                is_anon: true,
-                is_eval_always: false,
+            pub fn Null() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-                fingerprint_style: || FingerprintStyle::Unit,
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn TraitSelect() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: true,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            pub const CompileCodegenUnit: DepKindStruct = DepKindStruct {
-                is_anon: false,
-                is_eval_always: false,
+            pub fn CompileCodegenUnit() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-                fingerprint_style: || FingerprintStyle::Opaque,
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+            pub fn CompileMonoItem() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
 
-            pub const CompileMonoItem: DepKindStruct = DepKindStruct {
-                is_anon: false,
-                is_eval_always: false,
+            $(pub fn $name()-> DepKindStruct {
+                let is_anon = is_anon!([$($modifiers)*]);
+                let is_eval_always = is_eval_always!([$($modifiers)*]);
 
-                fingerprint_style: || FingerprintStyle::Opaque,
-                force_from_dep_node: |_, _| false,
-                try_load_from_on_disk_cache: |_, _| {},
-            };
+                let fingerprint_style =
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::fingerprint_style();
 
-            $(pub const $name: DepKindStruct = {
-                const is_anon: bool = is_anon!([$($modifiers)*]);
-                const is_eval_always: bool = is_eval_always!([$($modifiers)*]);
+                if is_anon || !fingerprint_style.reconstructible() {
+                    return DepKindStruct {
+                        is_anon,
+                        is_eval_always,
+                        fingerprint_style,
+                        force_from_dep_node: None,
+                        try_load_from_on_disk_cache: None,
+                    }
+                }
 
                 #[inline(always)]
-                fn fingerprint_style() -> FingerprintStyle {
-                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>
-                        ::fingerprint_style()
+                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option<query_keys::$name<'tcx>> {
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, &dep_node)
                 }
 
-                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<query_keys::$name<'tcx>> {
-                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
+                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool {
+                    if let Some(key) = recover(tcx, dep_node) {
+                        let tcx = QueryCtxt::from_tcx(tcx);
+                        force_query::<queries::$name<'_>, _>(tcx, key, dep_node);
+                        true
+                    } else {
+                        false
+                    }
                 }
 
-                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool {
-                    let tcx = QueryCtxt::from_tcx(tcx);
-                    force_query::<queries::$name<'_>, _>(tcx, dep_node)
-                }
+                fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) {
+                    debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
-                fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: &DepNode) {
+                    let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
                     let tcx = QueryCtxt::from_tcx(tcx);
-                    if is_anon {
-                        return
-                    }
-
-                    if !fingerprint_style().reconstructible() {
-                        return
-                    }
-
-                    debug_assert!(tcx.dep_graph.is_green(dep_node));
-
-                    let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
                     if queries::$name::cache_on_disk(tcx, &key, None) {
                         let _ = tcx.$name(key);
                     }
@@ -418,13 +427,15 @@ macro_rules! define_queries {
                     is_anon,
                     is_eval_always,
                     fingerprint_style,
-                    force_from_dep_node,
-                    try_load_from_on_disk_cache,
+                    force_from_dep_node: Some(force_from_dep_node),
+                    try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
                 }
-            };)*
+            })*
         }
 
-        pub static QUERY_CALLBACKS: &[DepKindStruct] = &make_dep_kind_array!(query_callbacks);
+        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
+            arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
+        }
     }
 }
 
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 228b4054967..503907e26d2 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -576,7 +576,7 @@ impl<K: DepKind> DepGraph<K> {
             "try_mark_previous_green({:?}) --- trying to force dependency {:?}",
             dep_node, dep_dep_node
         );
-        if !tcx.dep_context().try_force_from_dep_node(dep_dep_node) {
+        if !tcx.dep_context().try_force_from_dep_node(*dep_dep_node) {
             // The DepNode could not be forced.
             debug!(
                 "try_mark_previous_green({:?}) - END - dependency {:?} could not be forced",
@@ -749,7 +749,7 @@ impl<K: DepKind> DepGraph<K> {
             match data.colors.get(prev_index) {
                 Some(DepNodeColor::Green(_)) => {
                     let dep_node = data.previous.index_to_node(prev_index);
-                    tcx.try_load_from_on_disk_cache(&dep_node);
+                    tcx.try_load_from_on_disk_cache(dep_node);
                 }
                 None | Some(DepNodeColor::Red) => {
                     // We can skip red nodes because a node can only be marked
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index d7f7758d02c..047fc9f10cc 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -39,10 +39,10 @@ pub trait DepContext: Copy {
     fn fingerprint_style(&self, kind: Self::DepKind) -> FingerprintStyle;
 
     /// Try to force a dep node to execute and see if it's green.
-    fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
+    fn try_force_from_dep_node(&self, dep_node: DepNode<Self::DepKind>) -> bool;
 
     /// Load data from the on-disk cache.
-    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
+    fn try_load_from_on_disk_cache(&self, dep_node: DepNode<Self::DepKind>);
 }
 
 pub trait HasDepContext: Copy {
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index aa35347534d..d6fc4a669e7 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -665,41 +665,6 @@ where
     }
 }
 
-#[inline(never)]
-fn force_query_impl<CTX, C>(
-    tcx: CTX,
-    state: &QueryState<CTX::DepKind, C::Key>,
-    cache: &QueryCacheStore<C>,
-    key: C::Key,
-    dep_node: DepNode<CTX::DepKind>,
-    query: &QueryVtable<CTX, C::Key, C::Value>,
-    compute: fn(CTX::DepContext, C::Key) -> C::Value,
-) -> bool
-where
-    C: QueryCache,
-    C::Key: DepNodeParams<CTX::DepContext>,
-    CTX: QueryContext,
-{
-    debug_assert!(!query.anon);
-
-    // We may be concurrently trying both execute and force a query.
-    // Ensure that only one of them runs the query.
-    let cached = cache.cache.lookup(cache, &key, |_, index| {
-        if unlikely!(tcx.dep_context().profiler().enabled()) {
-            tcx.dep_context().profiler().query_cache_hit(index.into());
-        }
-    });
-
-    let lookup = match cached {
-        Ok(()) => return true,
-        Err(lookup) => lookup,
-    };
-
-    let _ =
-        try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), query, compute);
-    true
-}
-
 pub enum QueryMode {
     Get,
     Ensure,
@@ -747,34 +712,30 @@ where
     Some(result)
 }
 
-pub fn force_query<Q, CTX>(tcx: CTX, dep_node: &DepNode<CTX::DepKind>) -> bool
+pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, dep_node: DepNode<CTX::DepKind>)
 where
     Q: QueryDescription<CTX>,
     Q::Key: DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
-    if Q::ANON {
-        return false;
-    }
+    let query = &Q::VTABLE;
+    debug_assert!(!Q::ANON);
 
-    if !<Q::Key as DepNodeParams<CTX::DepContext>>::fingerprint_style().reconstructible() {
-        return false;
-    }
+    // We may be concurrently trying both execute and force a query.
+    // Ensure that only one of them runs the query.
+    let cache = Q::query_cache(tcx);
+    let cached = cache.cache.lookup(cache, &key, |_, index| {
+        if unlikely!(tcx.dep_context().profiler().enabled()) {
+            tcx.dep_context().profiler().query_cache_hit(index.into());
+        }
+    });
 
-    let Some(key) =
-        <Q::Key as DepNodeParams<CTX::DepContext>>::recover(*tcx.dep_context(), &dep_node)
-    else {
-        return false;
+    let lookup = match cached {
+        Ok(()) => return,
+        Err(lookup) => lookup,
     };
 
     let compute = Q::compute_fn(tcx, &key);
-    force_query_impl(
-        tcx,
-        Q::query_state(tcx),
-        Q::query_cache(tcx),
-        key,
-        *dep_node,
-        &Q::VTABLE,
-        compute,
-    )
+    let state = Q::query_state(tcx);
+    try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), query, compute);
 }