Auto merge of #109333 - Zoxc:erase-query-cache-values, r=cjgillot

Erase query cache values

This replaces most concrete query values `V` with `MaybeUninit<[u8; { size_of::<V>() }]>` without introducing dynamic dispatch like https://github.com/rust-lang/rust/pull/108638 does. This is split out of https://github.com/rust-lang/rust/pull/108638 so the performance impact of only this change can be measured.

r? `@cjgillot`

11 files changed, 465 insertions, 61 deletions

diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 0e883424fd4..b4edb02f6c4 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -33,6 +33,7 @@
 #![feature(generators)]
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
+#![feature(inline_const)]
 #![feature(iter_from_generator)]
 #![feature(local_key_cell_methods)]
 #![feature(negative_impls)]
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
new file mode 100644
index 00000000000..5462ced16d6
--- /dev/null
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -0,0 +1,337 @@
+use crate::mir;
+use crate::traits;
+use crate::ty::{self, Ty};
+use std::mem::{size_of, transmute_copy, MaybeUninit};
+
+#[derive(Copy, Clone)]
+pub struct Erased<T: Copy> {
+    // We use `MaybeUninit` here so we can store any value
+    // in `data` since we aren't actually storing a `T`.
+    data: MaybeUninit<T>,
+}
+
+pub trait EraseType: Copy {
+    type Result: Copy;
+}
+
+// Allow `type_alias_bounds` since compilation will fail without `EraseType`.
+#[allow(type_alias_bounds)]
+pub type Erase<T: EraseType> = Erased<impl Copy>;
+
+#[inline(always)]
+pub fn erase<T: EraseType>(src: T) -> Erase<T> {
+    // Ensure the sizes match
+    const {
+        if std::mem::size_of::<T>() != std::mem::size_of::<T::Result>() {
+            panic!("size of T must match erased type T::Result")
+        }
+    };
+
+    Erased::<<T as EraseType>::Result> {
+        // SAFETY: Is it safe to transmute to MaybeUninit for types with the same sizes.
+        data: unsafe { transmute_copy(&src) },
+    }
+}
+
+/// Restores an erased value.
+#[inline(always)]
+pub fn restore<T: EraseType>(value: Erase<T>) -> T {
+    let value: Erased<<T as EraseType>::Result> = value;
+    // SAFETY: Due to the use of impl Trait in `Erase` the only way to safetly create an instance
+    // of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of
+    // the right size.
+    unsafe { transmute_copy(&value.data) }
+}
+
+impl<T> EraseType for &'_ T {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T> EraseType for &'_ [T] {
+    type Result = [u8; size_of::<*const [()]>()];
+}
+
+impl<T> EraseType for &'_ ty::List<T> {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T> EraseType for Result<&'_ T, traits::query::NoSolution> {
+    type Result = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
+    type Result = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, traits::CodegenObligationError> {
+    type Result = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
+}
+
+impl<T> EraseType for Result<&'_ T, ty::layout::FnAbiError<'_>> {
+    type Result = [u8; size_of::<Result<&'static (), ty::layout::FnAbiError<'static>>>()];
+}
+
+impl<T> EraseType for Result<(&'_ T, rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed> {
+    type Result = [u8; size_of::<
+        Result<(&'static (), rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed>,
+    >()];
+}
+
+impl EraseType for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl EraseType for Result<Option<ty::Const<'_>>, rustc_errors::ErrorGuaranteed> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::Const<'static>>, rustc_errors::ErrorGuaranteed>>()];
+}
+
+impl EraseType for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
+    type Result = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
+}
+
+impl EraseType for Result<bool, ty::layout::LayoutError<'_>> {
+    type Result = [u8; size_of::<Result<bool, ty::layout::LayoutError<'static>>>()];
+}
+
+impl EraseType for Result<rustc_target::abi::TyAndLayout<'_, Ty<'_>>, ty::layout::LayoutError<'_>> {
+    type Result = [u8; size_of::<
+        Result<
+            rustc_target::abi::TyAndLayout<'static, Ty<'static>>,
+            ty::layout::LayoutError<'static>,
+        >,
+    >()];
+}
+
+impl EraseType for Result<ty::Const<'_>, mir::interpret::LitToConstError> {
+    type Result = [u8; size_of::<Result<ty::Const<'static>, mir::interpret::LitToConstError>>()];
+}
+
+impl EraseType for Result<mir::ConstantKind<'_>, mir::interpret::LitToConstError> {
+    type Result =
+        [u8; size_of::<Result<mir::ConstantKind<'static>, mir::interpret::LitToConstError>>()];
+}
+
+impl EraseType for Result<mir::interpret::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<
+        Result<mir::interpret::ConstAlloc<'static>, mir::interpret::ErrorHandled>,
+    >()];
+}
+
+impl EraseType for Result<mir::interpret::ConstValue<'_>, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<
+        Result<mir::interpret::ConstValue<'static>, mir::interpret::ErrorHandled>,
+    >()];
+}
+
+impl EraseType for Result<Option<ty::ValTree<'_>>, mir::interpret::ErrorHandled> {
+    type Result =
+        [u8; size_of::<Result<Option<ty::ValTree<'static>>, mir::interpret::ErrorHandled>>()];
+}
+
+impl EraseType for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
+    type Result =
+        [u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
+}
+
+impl<T> EraseType for Option<&'_ T> {
+    type Result = [u8; size_of::<Option<&'static ()>>()];
+}
+
+impl<T> EraseType for Option<&'_ [T]> {
+    type Result = [u8; size_of::<Option<&'static [()]>>()];
+}
+
+impl EraseType for Option<rustc_middle::hir::Owner<'_>> {
+    type Result = [u8; size_of::<Option<rustc_middle::hir::Owner<'static>>>()];
+}
+
+impl EraseType for Option<mir::DestructuredConstant<'_>> {
+    type Result = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
+}
+
+impl EraseType for Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
+    type Result = [u8; size_of::<Option<ty::EarlyBinder<ty::TraitRef<'static>>>>()];
+}
+
+impl EraseType for Option<ty::EarlyBinder<Ty<'_>>> {
+    type Result = [u8; size_of::<Option<ty::EarlyBinder<Ty<'static>>>>()];
+}
+
+impl<T> EraseType for rustc_hir::MaybeOwner<&'_ T> {
+    type Result = [u8; size_of::<rustc_hir::MaybeOwner<&'static ()>>()];
+}
+
+impl<T: EraseType> EraseType for ty::EarlyBinder<T> {
+    type Result = T::Result;
+}
+
+impl EraseType for ty::Binder<'_, ty::FnSig<'_>> {
+    type Result = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
+}
+
+impl<T0, T1> EraseType for (&'_ T0, &'_ T1) {
+    type Result = [u8; size_of::<(&'static (), &'static ())>()];
+}
+
+impl<T0, T1> EraseType for (&'_ T0, &'_ [T1]) {
+    type Result = [u8; size_of::<(&'static (), &'static [()])>()];
+}
+
+macro_rules! trivial {
+    ($($ty:ty),+ $(,)?) => {
+        $(
+            impl EraseType for $ty {
+                type Result = [u8; size_of::<$ty>()];
+            }
+        )*
+    }
+}
+
+trivial! {
+    (),
+    bool,
+    Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
+    Option<rustc_ast::expand::allocator::AllocatorKind>,
+    Option<rustc_attr::ConstStability>,
+    Option<rustc_attr::DefaultBodyStability>,
+    Option<rustc_attr::Stability>,
+    Option<rustc_data_structures::svh::Svh>,
+    Option<rustc_hir::def::DefKind>,
+    Option<rustc_hir::GeneratorKind>,
+    Option<rustc_hir::HirId>,
+    Option<rustc_middle::middle::stability::DeprecationEntry>,
+    Option<rustc_middle::ty::Destructor>,
+    Option<rustc_middle::ty::ImplTraitInTraitData>,
+    Option<rustc_span::def_id::CrateNum>,
+    Option<rustc_span::def_id::DefId>,
+    Option<rustc_span::def_id::LocalDefId>,
+    Option<rustc_span::Span>,
+    Option<rustc_target::spec::PanicStrategy>,
+    Option<usize>,
+    Result<(), rustc_errors::ErrorGuaranteed>,
+    Result<(), rustc_middle::traits::query::NoSolution>,
+    Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
+    rustc_ast::expand::allocator::AllocatorKind,
+    rustc_attr::ConstStability,
+    rustc_attr::DefaultBodyStability,
+    rustc_attr::Deprecation,
+    rustc_attr::Stability,
+    rustc_data_structures::svh::Svh,
+    rustc_errors::ErrorGuaranteed,
+    rustc_hir::Constness,
+    rustc_hir::def_id::DefId,
+    rustc_hir::def_id::DefIndex,
+    rustc_hir::def_id::LocalDefId,
+    rustc_hir::def::DefKind,
+    rustc_hir::Defaultness,
+    rustc_hir::definitions::DefKey,
+    rustc_hir::GeneratorKind,
+    rustc_hir::HirId,
+    rustc_hir::IsAsync,
+    rustc_hir::ItemLocalId,
+    rustc_hir::LangItem,
+    rustc_hir::OwnerId,
+    rustc_hir::Upvar,
+    rustc_index::bit_set::FiniteBitSet<u32>,
+    rustc_middle::metadata::ModChild,
+    rustc_middle::middle::dependency_format::Linkage,
+    rustc_middle::middle::exported_symbols::SymbolExportInfo,
+    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
+    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
+    rustc_middle::middle::stability::DeprecationEntry,
+    rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::interpret::AllocId,
+    rustc_middle::mir::interpret::ErrorHandled,
+    rustc_middle::mir::interpret::LitToConstError,
+    rustc_middle::thir::ExprId,
+    rustc_middle::traits::CodegenObligationError,
+    rustc_middle::traits::EvaluationResult,
+    rustc_middle::traits::OverflowError,
+    rustc_middle::traits::query::NoSolution,
+    rustc_middle::traits::WellFormedLoc,
+    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
+    rustc_middle::ty::AssocItem,
+    rustc_middle::ty::AssocItemContainer,
+    rustc_middle::ty::BoundVariableKind,
+    rustc_middle::ty::DeducedParamAttrs,
+    rustc_middle::ty::Destructor,
+    rustc_middle::ty::fast_reject::SimplifiedType,
+    rustc_middle::ty::ImplPolarity,
+    rustc_middle::ty::Representability,
+    rustc_middle::ty::ReprOptions,
+    rustc_middle::ty::UnusedGenericParams,
+    rustc_middle::ty::util::AlwaysRequiresDrop,
+    rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
+    rustc_session::config::CrateType,
+    rustc_session::config::EntryFnType,
+    rustc_session::config::OptLevel,
+    rustc_session::config::SymbolManglingVersion,
+    rustc_session::cstore::CrateDepKind,
+    rustc_session::cstore::ExternCrate,
+    rustc_session::cstore::LinkagePreference,
+    rustc_session::Limits,
+    rustc_session::lint::LintExpectationId,
+    rustc_span::def_id::CrateNum,
+    rustc_span::def_id::DefPathHash,
+    rustc_span::ExpnHash,
+    rustc_span::ExpnId,
+    rustc_span::Span,
+    rustc_span::Symbol,
+    rustc_span::symbol::Ident,
+    rustc_target::spec::PanicStrategy,
+    rustc_type_ir::Variance,
+    u32,
+    usize,
+}
+
+macro_rules! tcx_lifetime {
+    ($($($fake_path:ident)::+),+ $(,)?) => {
+        $(
+            impl<'tcx> EraseType for $($fake_path)::+<'tcx> {
+                type Result = [u8; size_of::<$($fake_path)::+<'static>>()];
+            }
+        )*
+    }
+}
+
+tcx_lifetime! {
+    rustc_middle::hir::Owner,
+    rustc_middle::middle::exported_symbols::ExportedSymbol,
+    rustc_middle::mir::ConstantKind,
+    rustc_middle::mir::DestructuredConstant,
+    rustc_middle::mir::interpret::ConstAlloc,
+    rustc_middle::mir::interpret::ConstValue,
+    rustc_middle::mir::interpret::GlobalId,
+    rustc_middle::mir::interpret::LitToConstInput,
+    rustc_middle::traits::ChalkEnvironmentAndGoal,
+    rustc_middle::traits::query::MethodAutoderefStepsResult,
+    rustc_middle::traits::query::type_op::AscribeUserType,
+    rustc_middle::traits::query::type_op::Eq,
+    rustc_middle::traits::query::type_op::ProvePredicate,
+    rustc_middle::traits::query::type_op::Subtype,
+    rustc_middle::ty::AdtDef,
+    rustc_middle::ty::AliasTy,
+    rustc_middle::ty::Clause,
+    rustc_middle::ty::ClosureTypeInfo,
+    rustc_middle::ty::Const,
+    rustc_middle::ty::DestructuredConst,
+    rustc_middle::ty::ExistentialTraitRef,
+    rustc_middle::ty::FnSig,
+    rustc_middle::ty::GenericArg,
+    rustc_middle::ty::GenericPredicates,
+    rustc_middle::ty::inhabitedness::InhabitedPredicate,
+    rustc_middle::ty::Instance,
+    rustc_middle::ty::InstanceDef,
+    rustc_middle::ty::layout::FnAbiError,
+    rustc_middle::ty::layout::LayoutError,
+    rustc_middle::ty::ParamEnv,
+    rustc_middle::ty::Predicate,
+    rustc_middle::ty::SymbolName,
+    rustc_middle::ty::TraitRef,
+    rustc_middle::ty::Ty,
+    rustc_middle::ty::UnevaluatedConst,
+    rustc_middle::ty::ValTree,
+    rustc_middle::ty::VtblEntry,
+}
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index d31b11f0892..96416896756 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -7,6 +7,7 @@
 use crate::ty::{self, print::describe_as_module, TyCtxt};
 use rustc_span::def_id::LOCAL_CRATE;
 
+pub mod erase;
 mod keys;
 pub use keys::{AsLocalKey, Key, LocalCrate};
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 30246fe4dbe..fa9fea72344 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -17,6 +17,7 @@ use crate::mir::interpret::{
 };
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
+use crate::query::erase::{erase, restore, Erase};
 use crate::query::{AsLocalKey, Key};
 use crate::thir;
 use crate::traits::query::{
@@ -57,6 +58,8 @@ use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
 use rustc_index::vec::IndexVec;
+pub(crate) use rustc_query_system::query::QueryJobId;
+use rustc_query_system::query::*;
 use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
 use rustc_session::cstore::{CrateDepKind, CrateSource};
 use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
@@ -66,18 +69,19 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi;
 use rustc_target::spec::PanicStrategy;
+
+use std::marker::PhantomData;
 use std::mem;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-pub(crate) use rustc_query_system::query::QueryJobId;
-use rustc_query_system::query::*;
-
 #[derive(Default)]
 pub struct QuerySystem<'tcx> {
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
+    // Since we erase query value types we tell the typesystem about them with `PhantomData`.
+    _phantom_values: QueryPhantomValues<'tcx>,
 }
 
 #[derive(Copy, Clone)]
@@ -263,8 +267,8 @@ macro_rules! define_callbacks {
                 pub fn $name<'tcx>(
                     _tcx: TyCtxt<'tcx>,
                     value: query_provided::$name<'tcx>,
-                ) -> query_values::$name<'tcx> {
-                    query_if_arena!([$($modifiers)*]
+                ) -> Erase<query_values::$name<'tcx>> {
+                    erase(query_if_arena!([$($modifiers)*]
                         {
                             if mem::needs_drop::<query_provided::$name<'tcx>>() {
                                 &*_tcx.query_system.arenas.$name.alloc(value)
@@ -273,7 +277,7 @@ macro_rules! define_callbacks {
                             }
                         }
                         (value)
-                    )
+                    ))
                 }
             )*
         }
@@ -282,7 +286,7 @@ macro_rules! define_callbacks {
             use super::*;
 
             $(
-                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
+                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache;
             )*
         }
 
@@ -335,6 +339,11 @@ macro_rules! define_callbacks {
         }
 
         #[derive(Default)]
+        pub struct QueryPhantomValues<'tcx> {
+            $($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
+        }
+
+        #[derive(Default)]
         pub struct QueryCaches<'tcx> {
             $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
         }
@@ -395,10 +404,10 @@ macro_rules! define_callbacks {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                restore::<$V>(match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(value) => value,
                     None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
-                }
+                })
             })*
         }
 
@@ -459,7 +468,7 @@ macro_rules! define_callbacks {
                 span: Span,
                 key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<$V>;)*
+            ) -> Option<Erase<$V>>;)*
         }
     };
 }
@@ -486,11 +495,13 @@ macro_rules! define_feedable {
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let value = query_provided_to_value::$name(tcx, value);
+                let erased = query_provided_to_value::$name(tcx, value);
+                let value = restore::<$V>(erased);
                 let cache = &tcx.query_system.caches.$name;
 
                 match try_get_cached(tcx, cache, &key) {
                     Some(old) => {
+                        let old = restore::<$V>(old);
                         bug!(
                             "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
                             stringify!($name),
@@ -505,7 +516,7 @@ macro_rules! define_feedable {
                             &value,
                             hash_result!([$($modifiers)*]),
                         );
-                        cache.complete(key, value, dep_node_index);
+                        cache.complete(key, erased, dep_node_index);
                         value
                     }
                 }
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 021a67c9513..7001a1eed57 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -18,19 +18,21 @@ extern crate rustc_middle;
 
 use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
-use rustc_middle::dep_graph::{self, DepKindStruct};
+use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
+use rustc_middle::query::erase::{erase, restore, Erase};
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
 };
 use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
 use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::SerializedDepNodeIndex;
+use rustc_query_system::Value;
 use rustc_span::Span;
 
 #[macro_use]
 mod plumbing;
 pub use plumbing::QueryCtxt;
-use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::query::*;
 #[cfg(parallel_compiler)]
 pub use rustc_query_system::query::{deadlock, QueryContext};
@@ -43,6 +45,13 @@ pub use on_disk_cache::OnDiskCache;
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
+/// This is implemented per query and restoring query values from their erased state.
+trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
+    type RestoredValue;
+
+    fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
+}
+
 rustc_query_append! { define_queries! }
 
 impl<'tcx> Queries<'tcx> {
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
index 35b7e5919e4..eec9dac7b9b 100644
--- a/compiler/rustc_query_impl/src/on_disk_cache.rs
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -1064,14 +1064,14 @@ impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
     }
 }
 
-pub fn encode_query_results<'a, 'tcx, Q>(
+pub(crate) fn encode_query_results<'a, 'tcx, Q>(
     query: Q,
     qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
 ) where
-    Q: super::QueryConfig<QueryCtxt<'tcx>>,
-    Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
+    Q: super::QueryConfigRestored<'tcx>,
+    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
 {
     let _timer = qcx
         .tcx
@@ -1089,7 +1089,7 @@ pub fn encode_query_results<'a, 'tcx, Q>(
 
             // Encode the type check tables with the `SerializedDepNodeIndex`
             // as tag.
-            encoder.encode_tagged(dep_node, value);
+            encoder.encode_tagged(dep_node, &Q::restore(*value));
         }
     });
 }
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 9bba26cc8e8..afbead7d1ae 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -263,14 +263,14 @@ macro_rules! feedable {
 }
 
 macro_rules! hash_result {
-    ([]) => {{
-        Some(dep_graph::hash_result)
+    ([][$V:ty]) => {{
+        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
     }};
-    ([(no_hash) $($rest:tt)*]) => {{
+    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
         None
     }};
-    ([$other:tt $($modifiers:tt)*]) => {
-        hash_result!([$($modifiers)*])
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        hash_result!([$($modifiers)*][$($args)*])
     };
 }
 
@@ -479,7 +479,7 @@ macro_rules! define_queries {
 
         $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
             type Key = query_keys::$name<'tcx>;
-            type Value = query_values::$name<'tcx>;
+            type Value = Erase<query_values::$name<'tcx>>;
 
             #[inline(always)]
             fn name(self) -> &'static str {
@@ -487,6 +487,11 @@ macro_rules! define_queries {
             }
 
             #[inline]
+            fn format_value(self) -> fn(&Self::Value) -> String {
+                |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
+            }
+
+            #[inline]
             fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
                 ::rustc_middle::query::cached::$name(tcx, key)
             }
@@ -508,7 +513,7 @@ macro_rules! define_queries {
             }
 
             fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
-                tcx.$name(key)
+                erase(tcx.$name(key))
             }
 
             #[inline]
@@ -558,6 +563,16 @@ macro_rules! define_queries {
                 })
             }
 
+            #[inline]
+            fn value_from_cycle_error(
+                self,
+                tcx: TyCtxt<'tcx>,
+                cycle: &[QueryInfo<DepKind>],
+            ) -> Self::Value {
+                let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
+                erase(result)
+            }
+
             #[inline(always)]
             fn anon(self) -> bool {
                 is_anon!([$($modifiers)*])
@@ -590,7 +605,16 @@ macro_rules! define_queries {
 
             #[inline(always)]
             fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
-                hash_result!([$($modifiers)*])
+                hash_result!([$($modifiers)*][query_values::$name<'tcx>])
+            }
+        })*
+
+        $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
+            type RestoredValue = query_values::$name<'tcx>;
+
+            #[inline(always)]
+            fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
+                restore::<query_values::$name<'tcx>>(value)
             }
         })*
 
@@ -708,7 +732,7 @@ macro_rules! define_queries {
                     )
                 },
                 encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index|
-                    $crate::on_disk_cache::encode_query_results(
+                    $crate::on_disk_cache::encode_query_results::<super::queries::$name<'tcx>>(
                         super::queries::$name::default(),
                         qcx,
                         encoder,
@@ -793,14 +817,14 @@ macro_rules! define_queries_struct {
 
             $($(#[$attr])*
             #[inline(always)]
-            #[tracing::instrument(level = "trace", skip(self, tcx), ret)]
+            #[tracing::instrument(level = "trace", skip(self, tcx))]
             fn $name(
                 &'tcx self,
                 tcx: TyCtxt<'tcx>,
                 span: Span,
-                key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
+                key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_values::$name<'tcx>> {
+            ) -> Option<Erase<query_values::$name<'tcx>>> {
                 let qcx = QueryCtxt { tcx, queries: self };
                 get_query(
                     queries::$name::default(),
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 2ff7de8cb9e..534d13b1ae0 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -538,7 +538,14 @@ impl<K: DepKind> DepGraph<K> {
             if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
                 let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
                 if let Some(dep_node_index) = dep_node_index {
-                    crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
+                    crate::query::incremental_verify_ich(
+                        cx,
+                        data,
+                        result,
+                        prev_index,
+                        hash_result,
+                        |value| format!("{:?}", value),
+                    );
 
                     #[cfg(debug_assertions)]
                     if hash_result.is_some() {
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index d3efc22a194..3ac8a852a4e 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -18,7 +18,7 @@ pub trait CacheSelector<'tcx, V> {
 
 pub trait QueryCache: Sized {
     type Key: Hash + Eq + Copy + Debug;
-    type Value: Copy + Debug;
+    type Value: Copy;
 
     /// Checks if the query is already computed and in the cache.
     fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
@@ -52,7 +52,7 @@ impl<K, V> Default for DefaultCache<K, V> {
 impl<K, V> QueryCache for DefaultCache<K, V>
 where
     K: Eq + Hash + Copy + Debug,
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = K;
     type Value = V;
@@ -120,7 +120,7 @@ impl<V> Default for SingleCache<V> {
 
 impl<V> QueryCache for SingleCache<V>
 where
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = ();
     type Value = V;
@@ -164,7 +164,7 @@ impl<K: Idx, V> Default for VecCache<K, V> {
 impl<K, V> QueryCache for VecCache<K, V>
 where
     K: Eq + Idx + Copy + Debug,
-    V: Copy + Debug,
+    V: Copy,
 {
     type Key = K;
     type Value = V;
diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index a0aeb812af9..c8d77938510 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -4,7 +4,7 @@ use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex};
 use crate::error::HandleCycleError;
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
-use crate::query::{QueryContext, QueryState};
+use crate::query::{QueryContext, QueryInfo, QueryState};
 
 use rustc_data_structures::fingerprint::Fingerprint;
 use std::fmt::Debug;
@@ -20,10 +20,12 @@ pub trait QueryConfig<Qcx: QueryContext>: Copy {
     // `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap,
     // but it isn't necessary.
     type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Copy + Debug;
-    type Value: Debug + Copy;
+    type Value: Copy;
 
     type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
 
+    fn format_value(self) -> fn(&Self::Value) -> String;
+
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
     where
@@ -45,6 +47,13 @@ pub trait QueryConfig<Qcx: QueryContext>: Copy {
 
     fn loadable_from_disk(self, qcx: Qcx, key: &Self::Key, idx: SerializedDepNodeIndex) -> bool;
 
+    /// Synthesize an error value to let compilation continue after a cycle.
+    fn value_from_cycle_error(
+        self,
+        tcx: Qcx::DepContext,
+        cycle: &[QueryInfo<Qcx::DepKind>],
+    ) -> Self::Value;
+
     fn anon(self) -> bool;
     fn eval_always(self) -> bool;
     fn depth_limit(self) -> bool;
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 519ea5ffed1..20310483d7e 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -11,7 +11,6 @@ use crate::query::job::QueryLatch;
 use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
 use crate::query::SerializedDepNodeIndex;
 use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
-use crate::values::Value;
 use crate::HandleCycleError;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
@@ -120,43 +119,45 @@ where
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<Qcx, R, D: DepKind>(
+fn mk_cycle<Q, Qcx>(
+    query: Q,
     qcx: Qcx,
-    cycle_error: CycleError<D>,
+    cycle_error: CycleError<Qcx::DepKind>,
     handler: HandleCycleError,
-) -> R
+) -> Q::Value
 where
-    Qcx: QueryContext + HasDepContext<DepKind = D>,
-    R: std::fmt::Debug + Value<Qcx::DepContext, Qcx::DepKind>,
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
 {
     let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
-    handle_cycle_error(*qcx.dep_context(), &cycle_error, error, handler)
+    handle_cycle_error(query, qcx, &cycle_error, error, handler)
 }
 
-fn handle_cycle_error<Tcx, V>(
-    tcx: Tcx,
-    cycle_error: &CycleError<Tcx::DepKind>,
+fn handle_cycle_error<Q, Qcx>(
+    query: Q,
+    qcx: Qcx,
+    cycle_error: &CycleError<Qcx::DepKind>,
     mut error: DiagnosticBuilder<'_, ErrorGuaranteed>,
     handler: HandleCycleError,
-) -> V
+) -> Q::Value
 where
-    Tcx: DepContext,
-    V: Value<Tcx, Tcx::DepKind>,
+    Q: QueryConfig<Qcx>,
+    Qcx: QueryContext,
 {
     use HandleCycleError::*;
     match handler {
         Error => {
             error.emit();
-            Value::from_cycle_error(tcx, &cycle_error.cycle)
+            query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
         }
         Fatal => {
             error.emit();
-            tcx.sess().abort_if_errors();
+            qcx.dep_context().sess().abort_if_errors();
             unreachable!()
         }
         DelayBug => {
             error.delay_as_bug();
-            Value::from_cycle_error(tcx, &cycle_error.cycle)
+            query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
         }
     }
 }
@@ -269,7 +270,7 @@ where
         &qcx.current_query_job(),
         span,
     );
-    (mk_cycle(qcx, error, query.handle_cycle_error()), None)
+    (mk_cycle(query, qcx, error, query.handle_cycle_error()), None)
 }
 
 #[inline(always)]
@@ -306,7 +307,7 @@ where
 
             (v, Some(index))
         }
-        Err(cycle) => (mk_cycle(qcx, cycle, query.handle_cycle_error()), None),
+        Err(cycle) => (mk_cycle(query, qcx, cycle, query.handle_cycle_error()), None),
     }
 }
 
@@ -410,7 +411,8 @@ where
         // get evaluated first, and re-feed the query.
         if let Some((cached_result, _)) = cache.lookup(&key) {
             panic!(
-                "fed query later has its value computed. The already cached value: {cached_result:?}"
+                "fed query later has its value computed. The already cached value: {}",
+                (query.format_value())(&cached_result)
             );
         }
     }
@@ -581,6 +583,7 @@ where
                     &result,
                     prev_dep_node_index,
                     query.hash_result(),
+                    query.format_value(),
                 );
             }
 
@@ -626,19 +629,21 @@ where
         &result,
         prev_dep_node_index,
         query.hash_result(),
+        query.format_value(),
     );
 
     Some((result, dep_node_index))
 }
 
 #[inline]
-#[instrument(skip(tcx, dep_graph_data, result, hash_result), level = "debug")]
-pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
+#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
+pub(crate) fn incremental_verify_ich<Tcx, V>(
     tcx: Tcx,
     dep_graph_data: &DepGraphData<Tcx::DepKind>,
     result: &V,
     prev_index: SerializedDepNodeIndex,
     hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
+    format_value: fn(&V) -> String,
 ) where
     Tcx: DepContext,
 {
@@ -653,7 +658,7 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
     let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
 
     if new_hash != old_hash {
-        incremental_verify_ich_failed(tcx, prev_index, result);
+        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(&result));
     }
 }
 
@@ -677,7 +682,7 @@ where
 fn incremental_verify_ich_failed<Tcx>(
     tcx: Tcx,
     prev_index: SerializedDepNodeIndex,
-    result: &dyn Debug,
+    result: &dyn Fn() -> String,
 ) where
     Tcx: DepContext,
 {
@@ -707,7 +712,7 @@ fn incremental_verify_ich_failed<Tcx>(
             run_cmd,
             dep_node: format!("{dep_node:?}"),
         });
-        panic!("Found unstable fingerprints for {dep_node:?}: {result:?}");
+        panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
     }
 
     INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));