Auto merge of #108638 - Zoxc:erase-query-values-map, r=cjgillot

Use dynamic dispatch for queries

This replaces most concrete query values `V` with `MaybeUninit<[u8; { size_of::<V>() }]>` reducing the code instantiated by queries. The compile time of `rustc_query_impl` is reduced by 27%. It is an alternative to https://github.com/rust-lang/rust/pull/107937 which uses unstable const generics while this uses a `EraseType` trait which maps query values to their erased variant.

This is achieved by introducing an `Erased` type which does sanity check with `cfg(debug_assertions)`. The query caches gets instantiated with these erased types leaving the code in `rustc_query_system` unaware of them. `rustc_query_system` is changed to use instances of `QueryConfig` so that `rustc_query_impl` can pass in `DynamicConfig` which holds a pointer to a virtual table.

<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check</td><td align="right">1.7055s</td><td align="right">1.6949s</td><td align="right"> -0.62%</td></tr><tr><td>🟣 <b>hyper</b>:check</td><td align="right">0.2547s</td><td align="right">0.2528s</td><td align="right"> -0.73%</td></tr><tr><td>🟣 <b>regex</b>:check</td><td align="right">0.9590s</td><td align="right">0.9553s</td><td align="right"> -0.39%</td></tr><tr><td>🟣 <b>syn</b>:check</td><td align="right">1.5457s</td><td align="right">1.5440s</td><td align="right"> -0.11%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check</td><td align="right">5.9092s</td><td align="right">5.9009s</td><td align="right"> -0.14%</td></tr><tr><td>Total</td><td align="right">10.3741s</td><td align="right">10.3479s</td><td align="right"> -0.25%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9960s</td><td align="right"> -0.40%</td></tr></table>

<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:initial</td><td align="right">2.0605s</td><td align="right">2.0575s</td><td align="right"> -0.15%</td></tr><tr><td>🟣 <b>hyper</b>:check:initial</td><td align="right">0.3218s</td><td align="right">0.3216s</td><td align="right"> -0.07%</td></tr><tr><td>🟣 <b>regex</b>:check:initial</td><td align="right">1.1848s</td><td align="right">1.1839s</td><td align="right"> -0.07%</td></tr><tr><td>🟣 <b>syn</b>:check:initial</td><td align="right">1.9409s</td><td align="right">1.9376s</td><td align="right"> -0.17%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:initial</td><td align="right">7.3105s</td><td align="right">7.2928s</td><td align="right"> -0.24%</td></tr><tr><td>Total</td><td align="right">12.8185s</td><td align="right">12.7935s</td><td align="right"> -0.20%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9986s</td><td align="right"> -0.14%</td></tr></table>

<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:unchanged</td><td align="right">0.4606s</td><td align="right">0.4617s</td><td align="right"> 0.24%</td></tr><tr><td>🟣 <b>hyper</b>:check:unchanged</td><td align="right">0.1335s</td><td align="right">0.1336s</td><td align="right"> 0.08%</td></tr><tr><td>🟣 <b>regex</b>:check:unchanged</td><td align="right">0.3324s</td><td align="right">0.3346s</td><td align="right"> 0.65%</td></tr><tr><td>🟣 <b>syn</b>:check:unchanged</td><td align="right">0.6268s</td><td align="right">0.6307s</td><td align="right"> 0.64%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:unchanged</td><td align="right">1.8248s</td><td align="right">1.8508s</td><td align="right">💔  1.43%</td></tr><tr><td>Total</td><td align="right">3.3779s</td><td align="right">3.4113s</td><td align="right"> 0.99%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">1.0061s</td><td align="right"> 0.61%</td></tr></table>

It's based on https://github.com/rust-lang/rust/pull/108167.

r? `@cjgillot`

2 files changed, 287 insertions, 162 deletions

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 82b335f4b4b..39ca4659541 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -3,11 +3,12 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref
 #![feature(const_mut_refs)]
+#![feature(const_refs_to_cell)]
 #![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
+#![allow(rustc::potential_query_instability, unused_parens)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -15,16 +16,27 @@
 extern crate rustc_middle;
 
 use crate::plumbing::{encode_all_query_results, try_mark_green};
+use field_offset::offset_of;
+use rustc_data_structures::stable_hasher::HashStable;
+use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
+use rustc_middle::dep_graph::DepNodeIndex;
 use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
 use rustc_middle::query::erase::{erase, restore, Erase};
+use rustc_middle::query::on_disk_cache::OnDiskCache;
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
+    DynamicQueries, DynamicQuery, ExternProviders, Providers, QueryCaches, QueryEngine,
+    QueryStates, QuerySystem, QuerySystemFns,
 };
-use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine, QuerySystemFns};
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
+use rustc_query_system::ich::StableHashingContext;
+use rustc_query_system::query::{
+    get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState,
+};
+use rustc_query_system::HandleCycleError;
 use rustc_query_system::Value;
 use rustc_span::Span;
 
@@ -32,31 +44,182 @@ use rustc_span::Span;
 mod plumbing;
 pub use crate::plumbing::QueryCtxt;
 
-pub use rustc_query_system::query::QueryConfig;
-use rustc_query_system::query::*;
-
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
-/// This is implemented per query and restoring query values from their erased state.
-trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
-    type RestoredValue;
+struct DynamicConfig<
+    'tcx,
+    C: QueryCache,
+    const ANON: bool,
+    const DEPTH_LIMIT: bool,
+    const FEEDABLE: bool,
+> {
+    dynamic: &'tcx DynamicQuery<'tcx, C>,
+}
 
-    fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
+    for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+{
+}
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
+    for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+{
+    fn clone(&self) -> Self {
+        DynamicConfig { dynamic: self.dynamic }
+    }
 }
 
-rustc_query_append! { define_queries! }
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
+    QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+where
+    for<'a> C::Key: HashStable<StableHashingContext<'a>>,
+{
+    type Key = C::Key;
+    type Value = C::Value;
+    type Cache = C;
+
+    #[inline(always)]
+    fn name(self) -> &'static str {
+        self.dynamic.name
+    }
+
+    #[inline(always)]
+    fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+        (self.dynamic.cache_on_disk)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn query_state<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, DepKind>
+    where
+        QueryCtxt<'tcx>: 'a,
+    {
+        self.dynamic.query_state.apply(&qcx.tcx.query_system.states)
+    }
+
+    #[inline(always)]
+    fn query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::Cache
+    where
+        'tcx: 'a,
+    {
+        self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
+    }
+
+    #[inline(always)]
+    fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.execute_query)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.compute)(qcx.tcx, key)
+    }
+
+    #[inline(always)]
+    fn try_load_from_disk(
+        self,
+        qcx: QueryCtxt<'tcx>,
+        key: &Self::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<Self::Value> {
+        if self.dynamic.can_load_from_disk {
+            (self.dynamic.try_load_from_disk)(qcx.tcx, key, prev_index, index)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn loadable_from_disk(
+        self,
+        qcx: QueryCtxt<'tcx>,
+        key: &Self::Key,
+        index: SerializedDepNodeIndex,
+    ) -> bool {
+        (self.dynamic.loadable_from_disk)(qcx.tcx, key, index)
+    }
+
+    fn value_from_cycle_error(
+        self,
+        tcx: TyCtxt<'tcx>,
+        cycle: &[QueryInfo<DepKind>],
+    ) -> Self::Value {
+        (self.dynamic.value_from_cycle_error)(tcx, cycle)
+    }
+
+    #[inline(always)]
+    fn format_value(self) -> fn(&Self::Value) -> String {
+        self.dynamic.format_value
+    }
+
+    #[inline(always)]
+    fn anon(self) -> bool {
+        ANON
+    }
+
+    #[inline(always)]
+    fn eval_always(self) -> bool {
+        self.dynamic.eval_always
+    }
+
+    #[inline(always)]
+    fn depth_limit(self) -> bool {
+        DEPTH_LIMIT
+    }
+
+    #[inline(always)]
+    fn feedable(self) -> bool {
+        FEEDABLE
+    }
+
+    #[inline(always)]
+    fn dep_kind(self) -> DepKind {
+        self.dynamic.dep_kind
+    }
+
+    #[inline(always)]
+    fn handle_cycle_error(self) -> HandleCycleError {
+        self.dynamic.handle_cycle_error
+    }
+
+    #[inline(always)]
+    fn hash_result(self) -> HashResult<Self::Value> {
+        self.dynamic.hash_result
+    }
+}
 
-pub fn query_system_fns<'tcx>(
+/// This is implemented per query. It allows restoring query values from their erased state
+/// and constructing a QueryConfig.
+trait QueryConfigRestored<'tcx> {
+    type RestoredValue;
+    type Config: QueryConfig<QueryCtxt<'tcx>>;
+
+    fn config(tcx: TyCtxt<'tcx>) -> Self::Config;
+    fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value)
+    -> Self::RestoredValue;
+}
+
+pub fn query_system<'tcx>(
     local_providers: Providers,
     extern_providers: ExternProviders,
-) -> QuerySystemFns<'tcx> {
-    QuerySystemFns {
-        engine: engine(),
-        local_providers,
-        extern_providers,
-        query_structs: make_dep_kind_array!(query_structs).to_vec(),
-        encode_query_results: encode_all_query_results,
-        try_mark_green: try_mark_green,
+    on_disk_cache: Option<OnDiskCache<'tcx>>,
+) -> QuerySystem<'tcx> {
+    QuerySystem {
+        states: Default::default(),
+        arenas: Default::default(),
+        caches: Default::default(),
+        dynamic_queries: dynamic_queries(),
+        on_disk_cache,
+        fns: QuerySystemFns {
+            engine: engine(),
+            local_providers,
+            extern_providers,
+            query_structs: make_dep_kind_array!(query_structs).to_vec(),
+            encode_query_results: encode_all_query_results,
+            try_mark_green: try_mark_green,
+        },
+        jobs: AtomicU64::new(1),
     }
 }
+
+rustc_query_append! { define_queries! }
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 9f8ac7ccd0b..74924e8113e 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -4,6 +4,7 @@
 
 use crate::rustc_middle::dep_graph::DepContext;
 use crate::rustc_middle::ty::TyEncoder;
+use crate::QueryConfigRestored;
 use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
 use rustc_data_structures::sync::Lock;
 use rustc_errors::Diagnostic;
@@ -265,14 +266,14 @@ macro_rules! hash_result {
 }
 
 macro_rules! call_provider {
-    ([][$qcx:expr, $name:ident, $key:expr]) => {{
-        ($qcx.query_system.fns.local_providers.$name)($qcx, $key)
+    ([][$tcx:expr, $name:ident, $key:expr]) => {{
+        ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
     }};
-    ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{
+    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
         if let Some(key) = $key.as_local_key() {
-            ($qcx.query_system.fns.local_providers.$name)($qcx, key)
+            ($tcx.query_system.fns.local_providers.$name)($tcx, key)
         } else {
-            ($qcx.query_system.fns.extern_providers.$name)($qcx, $key)
+            ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
         }
     }};
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
@@ -341,7 +342,7 @@ pub(crate) fn create_query_frame<
 }
 
 pub(crate) fn encode_query_results<'a, 'tcx, Q>(
-    query: Q,
+    query: Q::Config,
     qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
@@ -392,12 +393,26 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeI
 
 pub(crate) fn try_load_from_disk<'tcx, V>(
     tcx: TyCtxt<'tcx>,
-    id: SerializedDepNodeIndex,
+    prev_index: SerializedDepNodeIndex,
+    index: DepNodeIndex,
 ) -> Option<V>
 where
     V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
 {
-    tcx.query_system.on_disk_cache.as_ref()?.try_load_query_result(tcx, id)
+    let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
+
+    let prof_timer = tcx.prof.incr_cache_loading();
+
+    // The call to `with_query_deserialization` enforces that no new `DepNodes`
+    // are created during deserialization. See the docs of that method for more
+    // details.
+    let value = tcx
+        .dep_graph
+        .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
+
+    prof_timer.finish_with_query_invocation_id(index.into());
+
+    value
 }
 
 fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
@@ -434,10 +449,9 @@ where
 
 pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
 where
-    Q: QueryConfig<QueryCtxt<'tcx>> + Default,
-    Q::Key: DepNodeParams<TyCtxt<'tcx>>,
+    Q: QueryConfigRestored<'tcx>,
 {
-    let fingerprint_style = Q::Key::fingerprint_style();
+    let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
 
     if is_anon || !fingerprint_style.reconstructible() {
         return DepKindStruct {
@@ -453,9 +467,11 @@ where
         is_anon,
         is_eval_always,
         fingerprint_style,
-        force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)),
+        force_from_dep_node: Some(|tcx, dep_node| {
+            force_from_dep_node(Q::config(tcx), tcx, dep_node)
+        }),
         try_load_from_on_disk_cache: Some(|tcx, dep_node| {
-            try_load_from_on_disk_cache(Q::default(), tcx, dep_node)
+            try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
         }),
     }
 }
@@ -491,7 +507,7 @@ macro_rules! define_queries {
                     mode: QueryMode,
                 ) -> Option<Erase<query_values::$name<'tcx>>> {
                     get_query(
-                        queries::$name::default(),
+                        queries::$name::config(tcx),
                         QueryCtxt::new(tcx),
                         span,
                         key,
@@ -519,146 +535,91 @@ macro_rules! define_queries {
             )*
         }
 
-        $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
-            type Key = query_keys::$name<'tcx>;
-            type Value = Erase<query_values::$name<'tcx>>;
-
-            #[inline(always)]
-            fn name(self) -> &'static str {
-                stringify!($name)
-            }
-
-            #[inline]
-            fn format_value(self) -> fn(&Self::Value) -> String {
-                |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
-            }
-
-            #[inline]
-            fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
-                ::rustc_middle::query::cached::$name(tcx, key)
-            }
-
-            type Cache = query_storage::$name<'tcx>;
-
-            #[inline(always)]
-            fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind>
-                where QueryCtxt<'tcx>: 'a
-            {
-                &tcx.query_system.states.$name
-            }
-
-            #[inline(always)]
-            fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
-                where 'tcx:'a
-            {
-                &tcx.query_system.caches.$name
-            }
-
-            fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                erase(tcx.$name(key))
-            }
-
-            #[inline]
-            #[allow(unused_variables)]
-            fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                query_provided_to_value::$name(
-                    qcx.tcx,
-                    call_provider!([$($modifiers)*][qcx.tcx, $name, key])
-                )
-            }
+        #[allow(nonstandard_style)]
+        mod dynamic_query {
+            use super::*;
 
-            #[inline]
-            fn try_load_from_disk(
-                self,
-                _qcx: QueryCtxt<'tcx>,
-                _key: &Self::Key
-            ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> {
-                should_ever_cache_on_disk!([$($modifiers)*] {
-                    if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) {
-                        Some(|qcx: QueryCtxt<'tcx>, dep_node| {
-                            let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
-                                qcx.tcx,
-                                dep_node
-                            );
-                            value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
-                        })
-                    } else {
-                        None
+            $(
+                pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx>> {
+                    DynamicQuery {
+                        name: stringify!($name),
+                        eval_always: is_eval_always!([$($modifiers)*]),
+                        dep_kind: dep_graph::DepKind::$name,
+                        handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
+                        query_state: offset_of!(QueryStates<'tcx> => $name),
+                        query_cache: offset_of!(QueryCaches<'tcx> => $name),
+                        cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
+                        execute_query: |tcx, key| erase(tcx.$name(key)),
+                        compute: |tcx, key| query_provided_to_value::$name(
+                            tcx,
+                            call_provider!([$($modifiers)*][tcx, $name, key])
+                        ),
+                        can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
+                        try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
+                            |tcx, key, prev_index, index| {
+                                if ::rustc_middle::query::cached::$name(tcx, key) {
+                                    let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
+                                        tcx,
+                                        prev_index,
+                                        index,
+                                    );
+                                    value.map(|value| query_provided_to_value::$name(tcx, value))
+                                } else {
+                                    None
+                                }
+                            }
+                        } {
+                            |_tcx, _key, _prev_index, _index| None
+                        }),
+                        value_from_cycle_error: |tcx, cycle| {
+                            let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
+                            erase(result)
+                        },
+                        loadable_from_disk: |_tcx, _key, _index| {
+                            should_ever_cache_on_disk!([$($modifiers)*] {
+                                ::rustc_middle::query::cached::$name(_tcx, _key) &&
+                                    $crate::plumbing::loadable_from_disk(_tcx, _index)
+                            } {
+                                false
+                            })
+                        },
+                        hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]),
+                        format_value: |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value)),
                     }
-                } {
-                    None
-                })
-            }
-
-            #[inline]
-            fn loadable_from_disk(
-                self,
-                _qcx: QueryCtxt<'tcx>,
-                _key: &Self::Key,
-                _index: SerializedDepNodeIndex,
-            ) -> bool {
-                should_ever_cache_on_disk!([$($modifiers)*] {
-                    self.cache_on_disk(_qcx.tcx, _key) &&
-                        $crate::plumbing::loadable_from_disk(_qcx.tcx, _index)
-                } {
-                    false
-                })
-            }
-
-            #[inline]
-            fn value_from_cycle_error(
-                self,
-                tcx: TyCtxt<'tcx>,
-                cycle: &[QueryInfo<DepKind>],
-            ) -> Self::Value {
-                let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
-                erase(result)
-            }
-
-            #[inline(always)]
-            fn anon(self) -> bool {
-                is_anon!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn eval_always(self) -> bool {
-                is_eval_always!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn depth_limit(self) -> bool {
-                depth_limit!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn feedable(self) -> bool {
-                feedable!([$($modifiers)*])
-            }
+                }
+            )*
+        }
 
-            #[inline(always)]
-            fn dep_kind(self) -> rustc_middle::dep_graph::DepKind {
-                dep_graph::DepKind::$name
-            }
+        $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
+            type RestoredValue = query_values::$name<'tcx>;
+            type Config = DynamicConfig<
+                'tcx,
+                query_storage::$name<'tcx>,
+                { is_anon!([$($modifiers)*]) },
+                { depth_limit!([$($modifiers)*]) },
+                { feedable!([$($modifiers)*]) },
+            >;
 
             #[inline(always)]
-            fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError {
-                handle_cycle_error!([$($modifiers)*])
+            fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
+                DynamicConfig {
+                    dynamic: &tcx.query_system.dynamic_queries.$name,
+                }
             }
 
             #[inline(always)]
-            fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
-                hash_result!([$($modifiers)*][query_values::$name<'tcx>])
+            fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
+                restore::<query_values::$name<'tcx>>(value)
             }
         })*
 
-        $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
-            type RestoredValue = query_values::$name<'tcx>;
-
-            #[inline(always)]
-            fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
-                restore::<query_values::$name<'tcx>>(value)
+        pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
+            DynamicQueries {
+                $(
+                    $name: dynamic_query::$name(),
+                )*
             }
-        })*
+        }
 
         #[allow(nonstandard_style)]
         mod query_callbacks {
@@ -730,6 +691,7 @@ macro_rules! define_queries {
             use rustc_middle::ty::query::QueryStruct;
             use rustc_middle::ty::query::QueryKeyStringCache;
             use rustc_middle::dep_graph::DepKind;
+            use crate::QueryConfigRestored;
 
             pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> {
                 fn noop_try_collect_active_jobs(_: TyCtxt<'_>, _: &mut QueryMap<DepKind>) -> Option<()> {
@@ -774,7 +736,7 @@ macro_rules! define_queries {
                 },
                 encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index|
                     $crate::plumbing::encode_query_results::<super::queries::$name<'tcx>>(
-                        super::queries::$name::default(),
+                        super::queries::$name::config(tcx),
                         QueryCtxt::new(tcx),
                         encoder,
                         query_result_index,