Auto merge of #95573 - cjgillot:lower-query, r=michaelwoerister

Make lowering a query

Split from https://github.com/rust-lang/rust/pull/88186.

This PR refactors the relationship between lowering and the resolver outputs in order to make lowering itself a query.
In a first part, lowering is changed to avoid modifying resolver outputs, by maintaining its own data structures for creating new `NodeId`s and so.

Then, the `TyCtxt` is modified to allow creating new `LocalDefId`s from inside it. This is done by:
- enclosing `Definitions` in a lock, so as to allow modification;
- creating a query `register_def` whose purpose is to declare a `LocalDefId` to the query system.

See `TyCtxt::create_def` and `TyCtxt::iter_local_def_id` for more detailed explanations of the design.

7 files changed, 140 insertions, 73 deletions

diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 555baae35f5..2d095438fc4 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -183,6 +183,9 @@ rustc_dep_node_append!([define_dep_nodes!][ <'tcx>
     // We use this for most things when incr. comp. is turned off.
     [] Null,
 
+    // We use this to create a forever-red node.
+    [] Red,
+
     [anon] TraitSelect,
 
     // WARNING: if `Symbol` is changed, make sure you update `make_compile_codegen_unit` below.
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index e335cb395f8..c8b3b52b0fb 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -23,6 +23,7 @@ pub type EdgeFilter = rustc_query_system::dep_graph::debug::EdgeFilter<DepKind>;
 
 impl rustc_query_system::dep_graph::DepKind for DepKind {
     const NULL: Self = DepKind::Null;
+    const RED: Self = DepKind::Red;
 
     fn debug_node(node: &DepNode, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{:?}(", node.kind)?;
@@ -71,8 +72,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     type DepKind = DepKind;
 
     #[inline]
-    fn create_stable_hashing_context(&self) -> StableHashingContext<'_> {
-        TyCtxt::create_stable_hashing_context(*self)
+    fn with_stable_hashing_context<R>(&self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R {
+        TyCtxt::with_stable_hashing_context(*self, f)
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index cda0a60fa4e..06248320198 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -218,13 +218,6 @@ impl<'hir> Map<'hir> {
         self.tcx.local_def_id_to_hir_id(def_id)
     }
 
-    pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> + 'hir {
-        // Create a dependency to the crate to be sure we re-execute this when the amount of
-        // definitions change.
-        self.tcx.ensure().hir_crate(());
-        self.tcx.definitions_untracked().iter_local_def_id()
-    }
-
     /// Do not call this function directly. The query should be called.
     pub(super) fn opt_def_kind(self, local_def_id: LocalDefId) -> Option<DefKind> {
         let hir_id = self.local_def_id_to_hir_id(local_def_id);
@@ -1142,34 +1135,35 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, crate_num: CrateNum) -> Svh {
 
     source_file_names.sort_unstable();
 
-    let mut hcx = tcx.create_stable_hashing_context();
-    let mut stable_hasher = StableHasher::new();
-    hir_body_hash.hash_stable(&mut hcx, &mut stable_hasher);
-    upstream_crates.hash_stable(&mut hcx, &mut stable_hasher);
-    source_file_names.hash_stable(&mut hcx, &mut stable_hasher);
-    if tcx.sess.opts.debugging_opts.incremental_relative_spans {
-        let definitions = &tcx.definitions_untracked();
-        let mut owner_spans: Vec<_> = krate
-            .owners
-            .iter_enumerated()
-            .filter_map(|(def_id, info)| {
-                let _ = info.as_owner()?;
-                let def_path_hash = definitions.def_path_hash(def_id);
-                let span = resolutions.source_span[def_id];
-                debug_assert_eq!(span.parent(), None);
-                Some((def_path_hash, span))
-            })
-            .collect();
-        owner_spans.sort_unstable_by_key(|bn| bn.0);
-        owner_spans.hash_stable(&mut hcx, &mut stable_hasher);
-    }
-    tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher);
-    tcx.sess.local_stable_crate_id().hash_stable(&mut hcx, &mut stable_hasher);
-    // Hash visibility information since it does not appear in HIR.
-    resolutions.visibilities.hash_stable(&mut hcx, &mut stable_hasher);
-    resolutions.has_pub_restricted.hash_stable(&mut hcx, &mut stable_hasher);
+    let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| {
+        let mut stable_hasher = StableHasher::new();
+        hir_body_hash.hash_stable(&mut hcx, &mut stable_hasher);
+        upstream_crates.hash_stable(&mut hcx, &mut stable_hasher);
+        source_file_names.hash_stable(&mut hcx, &mut stable_hasher);
+        if tcx.sess.opts.debugging_opts.incremental_relative_spans {
+            let definitions = tcx.definitions_untracked();
+            let mut owner_spans: Vec<_> = krate
+                .owners
+                .iter_enumerated()
+                .filter_map(|(def_id, info)| {
+                    let _ = info.as_owner()?;
+                    let def_path_hash = definitions.def_path_hash(def_id);
+                    let span = resolutions.source_span[def_id];
+                    debug_assert_eq!(span.parent(), None);
+                    Some((def_path_hash, span))
+                })
+                .collect();
+            owner_spans.sort_unstable_by_key(|bn| bn.0);
+            owner_spans.hash_stable(&mut hcx, &mut stable_hasher);
+        }
+        tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher);
+        tcx.sess.local_stable_crate_id().hash_stable(&mut hcx, &mut stable_hasher);
+        // Hash visibility information since it does not appear in HIR.
+        resolutions.visibilities.hash_stable(&mut hcx, &mut stable_hasher);
+        resolutions.has_pub_restricted.hash_stable(&mut hcx, &mut stable_hasher);
+        stable_hasher.finish()
+    });
 
-    let crate_hash: Fingerprint = stable_hasher.finish();
     Svh::new(crate_hash.to_smaller_hash())
 }
 
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 12209d6725c..070a063c881 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -111,7 +111,6 @@ pub fn provide(providers: &mut Providers) {
         let hir = tcx.hir();
         hir.get_module_parent_node(hir.local_def_id_to_hir_id(id))
     };
-    providers.hir_crate = |tcx, ()| tcx.untracked_crate;
     providers.hir_crate_items = map::hir_crate_items;
     providers.crate_hash = map::crate_hash;
     providers.hir_module_items = map::hir_module_items;
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 57c4f3f3ba3..cbc45526e89 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -26,6 +26,12 @@ rustc_queries! {
         desc { "get the resolver outputs" }
     }
 
+    query resolver_for_lowering(_: ()) -> &'tcx Steal<ty::ResolverAstLowering> {
+        eval_always
+        no_hash
+        desc { "get the resolver for lowering" }
+    }
+
     /// Return the span for a definition.
     /// Contrary to `def_span` below, this query returns the full absolute span of the definition.
     /// This span is meant for dep-tracking rather than diagnostics. It should not be used outside
@@ -40,7 +46,8 @@ rustc_queries! {
     /// This is because the `hir_crate` query gives you access to all other items.
     /// To avoid this fate, do not call `tcx.hir().krate()`; instead,
     /// prefer wrappers like `tcx.visit_all_items_in_krate()`.
-    query hir_crate(key: ()) -> &'tcx Crate<'tcx> {
+    query hir_crate(key: ()) -> Crate<'tcx> {
+        storage(ArenaCacheSelector<'tcx>)
         eval_always
         desc { "get the crate HIR" }
     }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index af071b4e939..a594dab2e20 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -32,12 +32,13 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
-use rustc_data_structures::sync::{self, Lock, Lrc, WorkerLocal};
+use rustc_data_structures::sync::{self, Lock, Lrc, ReadGuard, RwLock, WorkerLocal};
 use rustc_data_structures::vec_map::VecMap;
 use rustc_errors::{DecorateLint, ErrorGuaranteed, LintDiagnosticBuilder, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
+use rustc_hir::definitions::Definitions;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{
@@ -1045,6 +1046,7 @@ impl<'tcx> Deref for TyCtxt<'tcx> {
 
 pub struct GlobalCtxt<'tcx> {
     pub arena: &'tcx WorkerLocal<Arena<'tcx>>,
+    pub hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
 
     interners: CtxtInterners<'tcx>,
 
@@ -1069,13 +1071,15 @@ pub struct GlobalCtxt<'tcx> {
     /// Common consts, pre-interned for your convenience.
     pub consts: CommonConsts<'tcx>,
 
-    definitions: rustc_hir::definitions::Definitions,
+    definitions: RwLock<Definitions>,
     cstore: Box<CrateStoreDyn>,
 
     /// Output of the resolver.
     pub(crate) untracked_resolutions: ty::ResolverOutputs,
-
-    pub(crate) untracked_crate: &'tcx hir::Crate<'tcx>,
+    untracked_resolver_for_lowering: Steal<ty::ResolverAstLowering>,
+    /// The entire crate as AST. This field serves as the input for the hir_crate query,
+    /// which lowers it from AST to HIR. It must not be read or used by anything else.
+    pub untracked_crate: Steal<Lrc<ast::Crate>>,
 
     /// This provides access to the incremental compilation on-disk cache for query results.
     /// Do not access this directly. It is only meant to be used by
@@ -1233,10 +1237,12 @@ impl<'tcx> TyCtxt<'tcx> {
         s: &'tcx Session,
         lint_store: Lrc<dyn Any + sync::Send + sync::Sync>,
         arena: &'tcx WorkerLocal<Arena<'tcx>>,
-        definitions: rustc_hir::definitions::Definitions,
+        hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
+        definitions: Definitions,
         cstore: Box<CrateStoreDyn>,
         untracked_resolutions: ty::ResolverOutputs,
-        krate: &'tcx hir::Crate<'tcx>,
+        untracked_resolver_for_lowering: ty::ResolverAstLowering,
+        krate: Lrc<ast::Crate>,
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
@@ -1263,16 +1269,18 @@ impl<'tcx> TyCtxt<'tcx> {
             sess: s,
             lint_store,
             arena,
+            hir_arena,
             interners,
             dep_graph,
-            definitions,
+            definitions: RwLock::new(definitions),
             cstore,
-            untracked_resolutions,
             prof: s.prof.clone(),
             types: common_types,
             lifetimes: common_lifetimes,
             consts: common_consts,
-            untracked_crate: krate,
+            untracked_resolutions,
+            untracked_resolver_for_lowering: Steal::new(untracked_resolver_for_lowering),
+            untracked_crate: Steal::new(krate),
             on_disk_cache,
             queries,
             query_caches: query::QueryCaches::default(),
@@ -1368,7 +1376,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_key(self, id: DefId) -> rustc_hir::definitions::DefKey {
         // Accessing the DefKey is ok, since it is part of DefPathHash.
         if let Some(id) = id.as_local() {
-            self.definitions.def_key(id)
+            self.definitions_untracked().def_key(id)
         } else {
             self.cstore.def_key(id)
         }
@@ -1382,7 +1390,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_path(self, id: DefId) -> rustc_hir::definitions::DefPath {
         // Accessing the DefPath is ok, since it is part of DefPathHash.
         if let Some(id) = id.as_local() {
-            self.definitions.def_path(id)
+            self.definitions_untracked().def_path(id)
         } else {
             self.cstore.def_path(id)
         }
@@ -1392,7 +1400,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_path_hash(self, def_id: DefId) -> rustc_hir::definitions::DefPathHash {
         // Accessing the DefPathHash is ok, it is incr. comp. stable.
         if let Some(def_id) = def_id.as_local() {
-            self.definitions.def_path_hash(def_id)
+            self.definitions_untracked().def_path_hash(def_id)
         } else {
             self.cstore.def_path_hash(def_id)
         }
@@ -1429,7 +1437,7 @@ impl<'tcx> TyCtxt<'tcx> {
         // If this is a DefPathHash from the local crate, we can look up the
         // DefId in the tcx's `Definitions`.
         if stable_crate_id == self.sess.local_stable_crate_id() {
-            self.definitions.local_def_path_hash_to_def_id(hash, err).to_def_id()
+            self.definitions.read().local_def_path_hash_to_def_id(hash, err).to_def_id()
         } else {
             // If this is a DefPathHash from an upstream crate, let the CrateStore map
             // it to a DefId.
@@ -1460,6 +1468,64 @@ impl<'tcx> TyCtxt<'tcx> {
         )
     }
 
+    /// Create a new definition within the incr. comp. engine.
+    pub fn create_def(self, parent: LocalDefId, data: hir::definitions::DefPathData) -> LocalDefId {
+        // This function modifies `self.definitions` using a side-effect.
+        // We need to ensure that these side effects are re-run by the incr. comp. engine.
+        // Depending on the forever-red node will tell the graph that the calling query
+        // needs to be re-evaluated.
+        use rustc_query_system::dep_graph::DepNodeIndex;
+        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+
+        // The following call has the side effect of modifying the tables inside `definitions`.
+        // These very tables are relied on by the incr. comp. engine to decode DepNodes and to
+        // decode the on-disk cache.
+        //
+        // Any LocalDefId which is used within queries, either as key or result, either:
+        // - has been created before the construction of the TyCtxt;
+        // - has been created by this call to `create_def`.
+        // As a consequence, this LocalDefId is always re-created before it is needed by the incr.
+        // comp. engine itself.
+        //
+        // This call also writes to the value of `source_span` and `expn_that_defined` queries.
+        // This is fine because:
+        // - those queries are `eval_always` so we won't miss their result changing;
+        // - this write will have happened before these queries are called.
+        self.definitions.write().create_def(parent, data)
+    }
+
+    pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> + 'tcx {
+        // Create a dependency to the crate to be sure we re-execute this when the amount of
+        // definitions change.
+        self.ensure().hir_crate(());
+        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
+        let definitions = self.definitions.leak();
+        definitions.iter_local_def_id()
+    }
+
+    pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
+        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // definitions change.
+        self.ensure().hir_crate(());
+        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
+        let definitions = self.definitions.leak();
+        definitions.def_path_table()
+    }
+
+    pub fn def_path_hash_to_def_index_map(
+        self,
+    ) -> &'tcx rustc_hir::def_path_hash_map::DefPathHashMap {
+        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // definitions change.
+        self.ensure().hir_crate(());
+        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
+        let definitions = self.definitions.leak();
+        definitions.def_path_hash_to_def_index_map()
+    }
+
     /// Note that this is *untracked* and should only be used within the query
     /// system if the result is otherwise tracked through queries
     pub fn cstore_untracked(self) -> &'tcx CrateStoreDyn {
@@ -1468,8 +1534,9 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Note that this is *untracked* and should only be used within the query
     /// system if the result is otherwise tracked through queries
-    pub fn definitions_untracked(self) -> &'tcx hir::definitions::Definitions {
-        &self.definitions
+    #[inline]
+    pub fn definitions_untracked(self) -> ReadGuard<'tcx, Definitions> {
+        self.definitions.read()
     }
 
     /// Note that this is *untracked* and should only be used within the query
@@ -1480,23 +1547,18 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline(always)]
-    pub fn create_stable_hashing_context(self) -> StableHashingContext<'tcx> {
-        StableHashingContext::new(
-            self.sess,
-            &self.definitions,
-            &*self.cstore,
-            &self.untracked_resolutions.source_span,
-        )
-    }
-
-    #[inline(always)]
-    pub fn create_no_span_stable_hashing_context(self) -> StableHashingContext<'tcx> {
-        StableHashingContext::ignore_spans(
+    pub fn with_stable_hashing_context<R>(
+        self,
+        f: impl FnOnce(StableHashingContext<'_>) -> R,
+    ) -> R {
+        let definitions = self.definitions_untracked();
+        let hcx = StableHashingContext::new(
             self.sess,
-            &self.definitions,
+            &*definitions,
             &*self.cstore,
             &self.untracked_resolutions.source_span,
-        )
+        );
+        f(hcx)
     }
 
     pub fn serialize_query_result_cache(self, encoder: FileEncoder) -> FileEncodeResult {
@@ -2304,7 +2366,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self.interners.intern_ty(
             st,
             self.sess,
-            &self.definitions,
+            &self.definitions.read(),
             &*self.cstore,
             // This is only used to create a stable hashing context.
             &self.untracked_resolutions.source_span,
@@ -2922,6 +2984,7 @@ fn ptr_eq<T, U>(t: *const T, u: *const U) -> bool {
 
 pub fn provide(providers: &mut ty::query::Providers) {
     providers.resolutions = |tcx, ()| &tcx.untracked_resolutions;
+    providers.resolver_for_lowering = |tcx, ()| &tcx.untracked_resolver_for_lowering;
     providers.module_reexports =
         |tcx, id| tcx.resolutions(()).reexport_map.get(&id).map(|v| &v[..]);
     providers.crate_name = |tcx, id| {
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 3a876df84c2..0e581d7f1f7 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -142,16 +142,16 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Creates a hash of the type `Ty` which will be the same no matter what crate
     /// context it's calculated within. This is used by the `type_id` intrinsic.
     pub fn type_id_hash(self, ty: Ty<'tcx>) -> u64 {
-        let mut hasher = StableHasher::new();
-        let mut hcx = self.create_stable_hashing_context();
-
         // We want the type_id be independent of the types free regions, so we
         // erase them. The erase_regions() call will also anonymize bound
         // regions, which is desirable too.
         let ty = self.erase_regions(ty);
 
-        hcx.while_hashing_spans(false, |hcx| ty.hash_stable(hcx, &mut hasher));
-        hasher.finish()
+        self.with_stable_hashing_context(|mut hcx| {
+            let mut hasher = StableHasher::new();
+            hcx.while_hashing_spans(false, |hcx| ty.hash_stable(hcx, &mut hasher));
+            hasher.finish()
+        })
     }
 
     pub fn res_generics_def_id(self, res: Res) -> Option<DefId> {