Auto merge of #108062 - Zoxc:spec-incr, r=cjgillot

Specialize query execution for incremental and non-incremental

This specializes query execution for incremental and non-incremental by passing in a separate `dyn QueryEngine` types, taking advantage of the virtual dispatch to avoid a branch. This ends up duplicating `try_execute_query`, hopefully the compile time cost of that is relatively low.

This is a performance improvement for the non-incremental path:
<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check</td><td align="right">1.8420s</td><td align="right">1.8331s</td><td align="right"> -0.48%</td></tr><tr><td>🟣 <b>hyper</b>:check</td><td align="right">0.2652s</td><td align="right">0.2631s</td><td align="right"> -0.78%</td></tr><tr><td>🟣 <b>regex</b>:check</td><td align="right">1.0161s</td><td align="right">1.0062s</td><td align="right"> -0.98%</td></tr><tr><td>🟣 <b>syn</b>:check</td><td align="right">1.6408s</td><td align="right">1.6197s</td><td align="right">💚  -1.28%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check</td><td align="right">6.3939s</td><td align="right">6.3558s</td><td align="right"> -0.60%</td></tr><tr><td>Total</td><td align="right">11.1580s</td><td align="right">11.0780s</td><td align="right"> -0.72%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9918s</td><td align="right"> -0.82%</td></tr></table>

The incremental path is more neutral:
<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:initial</td><td align="right">2.2210s</td><td align="right">2.2227s</td><td align="right"> 0.08%</td></tr><tr><td>🟣 <b>hyper</b>:check:initial</td><td align="right">0.3441s</td><td align="right">0.3443s</td><td align="right"> 0.05%</td></tr><tr><td>🟣 <b>regex</b>:check:initial</td><td align="right">1.2919s</td><td align="right">1.2877s</td><td align="right"> -0.33%</td></tr><tr><td>🟣 <b>syn</b>:check:initial</td><td align="right">2.0749s</td><td align="right">2.0721s</td><td align="right"> -0.14%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:initial</td><td align="right">7.9266s</td><td align="right">7.9206s</td><td align="right"> -0.07%</td></tr><tr><td>Total</td><td align="right">13.8585s</td><td align="right">13.8474s</td><td align="right"> -0.08%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9992s</td><td align="right"> -0.08%</td></tr></table>

r? `@cjgillot`

2 files changed, 37 insertions, 7 deletions

diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 5ca102a0946..3c542821b67 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -34,7 +34,8 @@ use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{
-    get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState,
+    get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap,
+    QueryMode, QueryState,
 };
 use rustc_query_system::HandleCycleError;
 use rustc_query_system::Value;
@@ -203,6 +204,7 @@ pub fn query_system<'tcx>(
     local_providers: Providers,
     extern_providers: ExternProviders,
     on_disk_cache: Option<OnDiskCache<'tcx>>,
+    incremental: bool,
 ) -> QuerySystem<'tcx> {
     QuerySystem {
         states: Default::default(),
@@ -211,7 +213,7 @@ pub fn query_system<'tcx>(
         dynamic_queries: dynamic_queries(),
         on_disk_cache,
         fns: QuerySystemFns {
-            engine: engine(),
+            engine: engine(incremental),
             local_providers,
             extern_providers,
             query_structs: make_dep_kind_array!(query_structs).to_vec(),
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 74924e8113e..5acfeb47a12 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -494,7 +494,7 @@ macro_rules! define_queries {
     (
      $($(#[$attr:meta])*
         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
-        mod get_query {
+        mod get_query_incr {
             use super::*;
 
             $(
@@ -506,7 +506,7 @@ macro_rules! define_queries {
                     key: query_keys::$name<'tcx>,
                     mode: QueryMode,
                 ) -> Option<Erase<query_values::$name<'tcx>>> {
-                    get_query(
+                    get_query_incr(
                         queries::$name::config(tcx),
                         QueryCtxt::new(tcx),
                         span,
@@ -517,9 +517,37 @@ macro_rules! define_queries {
             )*
         }
 
-        pub(crate) fn engine() -> QueryEngine {
-            QueryEngine {
-                $($name: get_query::$name,)*
+        mod get_query_non_incr {
+            use super::*;
+
+            $(
+                #[inline(always)]
+                #[tracing::instrument(level = "trace", skip(tcx))]
+                pub(super) fn $name<'tcx>(
+                    tcx: TyCtxt<'tcx>,
+                    span: Span,
+                    key: query_keys::$name<'tcx>,
+                    __mode: QueryMode,
+                ) -> Option<Erase<query_values::$name<'tcx>>> {
+                    Some(get_query_non_incr(
+                        queries::$name::config(tcx),
+                        QueryCtxt::new(tcx),
+                        span,
+                        key,
+                    ))
+                }
+            )*
+        }
+
+        pub(crate) fn engine(incremental: bool) -> QueryEngine {
+            if incremental {
+                QueryEngine {
+                    $($name: get_query_incr::$name,)*
+                }
+            } else {
+                QueryEngine {
+                    $($name: get_query_non_incr::$name,)*
+                }
             }
         }