Auto merge of #109333 - Zoxc:erase-query-cache-values, r=cjgillot

Erase query cache values

This replaces most concrete query values `V` with `MaybeUninit<[u8; { size_of::<V>() }]>` without introducing dynamic dispatch like https://github.com/rust-lang/rust/pull/108638 does. This is split out of https://github.com/rust-lang/rust/pull/108638 so the performance impact of only this change can be measured.

r? `@cjgillot`

3 files changed, 51 insertions, 18 deletions

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 021a67c9513..7001a1eed57 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -18,19 +18,21 @@ extern crate rustc_middle;
 
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
-use rustc_middle::dep_graph::{self, DepKindStruct};
+use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
+use rustc_middle::query::erase::{erase, restore, Erase};
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
 };
 use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
 use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::SerializedDepNodeIndex;
+use rustc_query_system::Value;
 use rustc_span::Span;
 
 #[macro_use]
 mod plumbing;
 pub use plumbing::QueryCtxt;
-use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::*;
 #[cfg(parallel_compiler)]
 pub use rustc_query_system::query::{deadlock, QueryContext};
@@ -43,6 +45,13 @@ pub use on_disk_cache::OnDiskCache;
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
+/// This is implemented per query and restoring query values from their erased state.
+trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
+    type RestoredValue;
+
+    fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
+}
+
 rustc_query_append! { define_queries! }
 
 impl<'tcx> Queries<'tcx> {
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
index 35b7e5919e4..eec9dac7b9b 100644
--- a/compiler/rustc_query_impl/src/on_disk_cache.rs
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -1064,14 +1064,14 @@ impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
     }
 }
 
-pub fn encode_query_results<'a, 'tcx, Q>(
+pub(crate) fn encode_query_results<'a, 'tcx, Q>(
     query: Q,
     qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
 ) where
-    Q: super::QueryConfig<QueryCtxt<'tcx>>,
-    Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
+    Q: super::QueryConfigRestored<'tcx>,
+    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
 {
     let _timer = qcx
         .tcx
@@ -1089,7 +1089,7 @@ pub fn encode_query_results<'a, 'tcx, Q>(
 
             // Encode the type check tables with the `SerializedDepNodeIndex`
             // as tag.
-            encoder.encode_tagged(dep_node, value);
+            encoder.encode_tagged(dep_node, &Q::restore(*value));
         }
     });
 }
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 9bba26cc8e8..afbead7d1ae 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -263,14 +263,14 @@ macro_rules! feedable {
 }
 
 macro_rules! hash_result {
-    ([]) => {{
-        Some(dep_graph::hash_result)
+    ([][$V:ty]) => {{
+        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
     }};
-    ([(no_hash) $($rest:tt)*]) => {{
+    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
         None
     }};
-    ([$other:tt $($modifiers:tt)*]) => {
-        hash_result!([$($modifiers)*])
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        hash_result!([$($modifiers)*][$($args)*])
     };
 }
 
@@ -479,7 +479,7 @@ macro_rules! define_queries {
 
         $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
             type Key = query_keys::$name<'tcx>;
-            type Value = query_values::$name<'tcx>;
+            type Value = Erase<query_values::$name<'tcx>>;
 
             #[inline(always)]
             fn name(self) -> &'static str {
@@ -487,6 +487,11 @@ macro_rules! define_queries {
             }
 
             #[inline]
+            fn format_value(self) -> fn(&Self::Value) -> String {
+                |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
+            }
+
+            #[inline]
             fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
                 ::rustc_middle::query::cached::$name(tcx, key)
             }
@@ -508,7 +513,7 @@ macro_rules! define_queries {
             }
 
             fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                tcx.$name(key)
+                erase(tcx.$name(key))
             }
 
             #[inline]
@@ -558,6 +563,16 @@ macro_rules! define_queries {
                 })
             }
 
+            #[inline]
+            fn value_from_cycle_error(
+                self,
+                tcx: TyCtxt<'tcx>,
+                cycle: &[QueryInfo<DepKind>],
+            ) -> Self::Value {
+                let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
+                erase(result)
+            }
+
             #[inline(always)]
             fn anon(self) -> bool {
                 is_anon!([$($modifiers)*])
@@ -590,7 +605,16 @@ macro_rules! define_queries {
 
             #[inline(always)]
             fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
-                hash_result!([$($modifiers)*])
+                hash_result!([$($modifiers)*][query_values::$name<'tcx>])
+            }
+        })*
+
+        $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
+            type RestoredValue = query_values::$name<'tcx>;
+
+            #[inline(always)]
+            fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
+                restore::<query_values::$name<'tcx>>(value)
             }
         })*
 
@@ -708,7 +732,7 @@ macro_rules! define_queries {
                     )
                 },
                 encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index|
-                    $crate::on_disk_cache::encode_query_results(
+                    $crate::on_disk_cache::encode_query_results::<super::queries::$name<'tcx>>(
                         super::queries::$name::default(),
                         qcx,
                         encoder,
@@ -793,14 +817,14 @@ macro_rules! define_queries_struct {
 
             $($(#[$attr])*
             #[inline(always)]
-            #[tracing::instrument(level = "trace", skip(self, tcx), ret)]
+            #[tracing::instrument(level = "trace", skip(self, tcx))]
             fn $name(
                 &'tcx self,
                 tcx: TyCtxt<'tcx>,
                 span: Span,
-                key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
+                key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_values::$name<'tcx>> {
+            ) -> Option<Erase<query_values::$name<'tcx>>> {
                 let qcx = QueryCtxt { tcx, queries: self };
                 get_query(
                     queries::$name::default(),