Auto merge of #152791 - nnethercote:rm-const-FLAGS, r=Zalathar

Remove `const FLAGS`.

*[View all comments](https://triagebot.infra.rust-lang.org/gh-comments/rust-lang/rust/pull/152791)*

The performance wins provided by these types are meagre, and I don't think they justify the code complexity they introduce.

r? @Zalathar

compiler/rustc_middle/src/query/plumbing.rs

@@ -1,4 +1,4 @@
-use std::fmt::Debug;
+use std::fmt;
 use std::ops::Deref;
 
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -12,7 +12,7 @@
 pub use sealed::IntoQueryParam;
 
 use crate::dep_graph;
-use crate::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex};
+use crate::dep_graph::{DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex};
 use crate::ich::StableHashingContext;
 use crate::queries::{
     ExternProviders, PerQueryVTables, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
@@ -104,7 +104,16 @@ pub enum QueryMode {
 /// Stores function pointers and other metadata for a particular query.
 pub struct QueryVTable<'tcx, C: QueryCache> {
     pub name: &'static str,
+
+    /// True if this query has the `anon` modifier.
+    pub anon: bool,
+    /// True if this query has the `eval_always` modifier.
     pub eval_always: bool,
+    /// True if this query has the `depth_limit` modifier.
+    pub depth_limit: bool,
+    /// True if this query has the `feedable` modifier.
+    pub feedable: bool,
+
     pub dep_kind: DepKind,
     /// How this query deals with query cycle errors.
     pub cycle_error_handling: CycleErrorHandling,
@@ -142,6 +151,85 @@ pub struct QueryVTable<'tcx, C: QueryCache> {
     pub description_fn: fn(TyCtxt<'tcx>, C::Key) -> String,
 }
 
+impl<'tcx, C: QueryCache> fmt::Debug for QueryVTable<'tcx, C> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // When debug-printing a query vtable (e.g. for ICE or tracing),
+        // just print the query name to know what query we're dealing with.
+        // The other fields and flags are probably just unhelpful noise.
+        //
+        // If there is need for a more detailed dump of all flags and fields,
+        // consider writing a separate dump method and calling it explicitly.
+        f.write_str(self.name)
+    }
+}
+
+impl<'tcx, C: QueryCache> QueryVTable<'tcx, C> {
+    #[inline(always)]
+    pub fn will_cache_on_disk_for_key(&self, tcx: TyCtxt<'tcx>, key: &C::Key) -> bool {
+        self.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key))
+    }
+
+    // Don't use this method to access query results, instead use the methods on TyCtxt.
+    #[inline(always)]
+    pub fn query_state(&self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> {
+        // Safety:
+        // This is just manually doing the subfield referencing through pointer math.
+        unsafe {
+            &*(&tcx.query_system.states as *const QueryStates<'tcx>)
+                .byte_add(self.query_state)
+                .cast::<QueryState<'tcx, C::Key>>()
+        }
+    }
+
+    // Don't use this method to access query results, instead use the methods on TyCtxt.
+    #[inline(always)]
+    pub fn query_cache(&self, tcx: TyCtxt<'tcx>) -> &'tcx C {
+        // Safety:
+        // This is just manually doing the subfield referencing through pointer math.
+        unsafe {
+            &*(&tcx.query_system.caches as *const QueryCaches<'tcx>)
+                .byte_add(self.query_cache)
+                .cast::<C>()
+        }
+    }
+
+    #[inline(always)]
+    pub fn try_load_from_disk(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        key: &C::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<C::Value> {
+        // `?` will return None immediately for queries that never cache to disk.
+        self.try_load_from_disk_fn?(tcx, key, prev_index, index)
+    }
+
+    #[inline]
+    pub fn is_loadable_from_disk(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        key: &C::Key,
+        index: SerializedDepNodeIndex,
+    ) -> bool {
+        self.is_loadable_from_disk_fn.map_or(false, |f| f(tcx, key, index))
+    }
+
+    /// Synthesize an error value to let compilation continue after a cycle.
+    pub fn value_from_cycle_error(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        cycle_error: &CycleError,
+        guar: ErrorGuaranteed,
+    ) -> C::Value {
+        (self.value_from_cycle_error)(tcx, cycle_error, guar)
+    }
+
+    pub fn construct_dep_node(&self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode {
+        DepNode::construct(tcx, self.dep_kind, key)
+    }
+}
+
 pub struct QuerySystemFns {
     pub engine: QueryEngine,
     pub local_providers: Providers,

compiler/rustc_query_impl/src/dep_kind_vtables.rs

@@ -2,8 +2,8 @@
 use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle};
 use rustc_middle::query::QueryCache;
 
+use crate::QueryDispatcherUnerased;
 use crate::plumbing::{force_from_dep_node_inner, try_load_from_on_disk_cache_inner};
-use crate::{QueryDispatcherUnerased, QueryFlags};
 
 /// [`DepKindVTable`] constructors for special dep kinds that aren't queries.
 #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")]
@@ -102,14 +102,14 @@ pub(crate) fn Metadata<'tcx>() -> DepKindVTable<'tcx> {
 
 /// Shared implementation of the [`DepKindVTable`] constructor for queries.
 /// Called from macro-generated code for each query.
-pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache, const FLAGS: QueryFlags>(
+pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache>(
+    is_anon: bool,
     is_eval_always: bool,
 ) -> DepKindVTable<'tcx>
 where
-    Q: QueryDispatcherUnerased<'tcx, Cache, FLAGS>,
+    Q: QueryDispatcherUnerased<'tcx, Cache>,
     Cache: QueryCache + 'tcx,
 {
-    let is_anon = FLAGS.is_anon;
     let key_fingerprint_style = if is_anon {
         KeyFingerprintStyle::Opaque
     } else {
@@ -131,10 +131,10 @@ pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q, Cache, const FLAGS: QueryF
         is_eval_always,
         key_fingerprint_style,
         force_from_dep_node: Some(|tcx, dep_node, _| {
-            force_from_dep_node_inner(Q::query_dispatcher(tcx), tcx, dep_node)
+            force_from_dep_node_inner(Q::query_vtable(tcx), tcx, dep_node)
         }),
         try_load_from_on_disk_cache: Some(|tcx, dep_node| {
-            try_load_from_on_disk_cache_inner(Q::query_dispatcher(tcx), tcx, dep_node)
+            try_load_from_on_disk_cache_inner(Q::query_vtable(tcx), tcx, dep_node)
         }),
     }
 }

compiler/rustc_query_impl/src/execution.rs

@@ -6,6 +6,7 @@
 use rustc_data_structures::{outline, sharded, sync};
 use rustc_errors::{Diag, FatalError, StashKey};
 use rustc_middle::dep_graph::{DepGraphData, DepNodeKey};
+use rustc_middle::query::plumbing::QueryVTable;
 use rustc_middle::query::{
     ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch,
     QueryMode, QueryStackDeferred, QueryStackFrame, QueryState,
@@ -19,7 +20,6 @@
 use crate::plumbing::{
     collect_active_jobs_from_all_queries, current_query_job, next_job_id, start_query,
 };
-use crate::{QueryFlags, SemiDynamicQueryDispatcher};
 
 #[inline]
 fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
@@ -102,8 +102,8 @@ struct ActiveJobGuard<'tcx, K>
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn mk_cycle<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     cycle_error: CycleError,
 ) -> C::Value {
@@ -111,13 +111,13 @@ fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
     handle_cycle_error(query, tcx, &cycle_error, error)
 }
 
-fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn handle_cycle_error<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     cycle_error: &CycleError,
     error: Diag<'_>,
 ) -> C::Value {
-    match query.cycle_error_handling() {
+    match query.cycle_error_handling {
         CycleErrorHandling::Error => {
             let guar = error.emit();
             query.value_from_cycle_error(tcx, cycle_error, guar)
@@ -208,8 +208,8 @@ fn drop(&mut self) {
 
 #[cold]
 #[inline(never)]
-fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn cycle_error<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     try_execute: QueryJobId,
     span: Span,
@@ -225,8 +225,8 @@ fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 }
 
 #[inline(always)]
-fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn wait_for_query<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     span: Span,
     key: C::Key,
@@ -255,7 +255,7 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
                         Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(),
                         _ => panic!(
                             "query '{}' result must be in the cache or the query must be poisoned after a wait",
-                            query.name()
+                            query.name
                         ),
                     }
                 })
@@ -271,8 +271,8 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 }
 
 #[inline(never)]
-fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn try_execute_query<'tcx, C: QueryCache, const INCR: bool>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     span: Span,
     key: C::Key,
@@ -308,7 +308,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
             // Drop the lock before we start executing the query
             drop(state_lock);
 
-            execute_job::<C, FLAGS, INCR>(query, tcx, state, key, key_hash, id, dep_node)
+            execute_job::<C, INCR>(query, tcx, state, key, key_hash, id, dep_node)
         }
         Entry::Occupied(mut entry) => {
             match &mut entry.get_mut().1 {
@@ -337,8 +337,8 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
 }
 
 #[inline(always)]
-fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn execute_job<'tcx, C: QueryCache, const INCR: bool>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     state: &'tcx QueryState<'tcx, C::Key>,
     key: C::Key,
@@ -360,25 +360,25 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
     };
 
     let cache = query.query_cache(tcx);
-    if query.feedable() {
+    if query.feedable {
         // We should not compute queries that also got a value via feeding.
         // This can't happen, as query feeding adds the very dependencies to the fed query
         // as its feeding query had. So if the fed query is red, so is its feeder, which will
         // get evaluated first, and re-feed the query.
         if let Some((cached_result, _)) = cache.lookup(&key) {
-            let Some(hasher) = query.hash_result() else {
+            let Some(hasher) = query.hash_result else {
                 panic!(
                     "no_hash fed query later has its value computed.\n\
                     Remove `no_hash` modifier to allow recomputation.\n\
                     The already cached value: {}",
-                    (query.format_value())(&cached_result)
+                    (query.format_value)(&cached_result)
                 );
             };
 
             let (old_hash, new_hash) = tcx.with_stable_hashing_context(|mut hcx| {
                 (hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result))
             });
-            let formatter = query.format_value();
+            let formatter = query.format_value;
             if old_hash != new_hash {
                 // We have an inconsistency. This can happen if one of the two
                 // results is tainted by errors.
@@ -386,7 +386,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
                     tcx.dcx().has_errors().is_some(),
                     "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
                         computed={:#?}\nfed={:#?}",
-                    query.dep_kind(),
+                    query.dep_kind,
                     key,
                     formatter(&result),
                     formatter(&cached_result),
@@ -403,8 +403,8 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
 
 // Fast path for when incr. comp. is off.
 #[inline(always)]
-fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn execute_job_non_incr<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: C::Key,
     job_id: QueryJobId,
@@ -419,14 +419,15 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 
     let prof_timer = tcx.prof.query_provider();
     // Call the query provider.
-    let result = start_query(tcx, job_id, query.depth_limit(), || query.invoke_provider(tcx, key));
+    let result =
+        start_query(tcx, job_id, query.depth_limit, || (query.invoke_provider_fn)(tcx, key));
     let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
     // Similarly, fingerprint the result to assert that
     // it doesn't have anything not considered hashable.
     if cfg!(debug_assertions)
-        && let Some(hash_result) = query.hash_result()
+        && let Some(hash_result) = query.hash_result
     {
         tcx.with_stable_hashing_context(|mut hcx| {
             hash_result(&mut hcx, &result);
@@ -437,8 +438,8 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 }
 
 #[inline(always)]
-fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn execute_job_incr<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: C::Key,
     mut dep_node_opt: Option<DepNode>,
@@ -447,7 +448,7 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
     let dep_graph_data =
         tcx.dep_graph.data().expect("should always be present in incremental mode");
 
-    if !query.anon() && !query.eval_always() {
+    if !query.anon && !query.eval_always {
         // `to_dep_node` is expensive for some `DepKind`s.
         let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(tcx, &key));
 
@@ -462,11 +463,12 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 
     let prof_timer = tcx.prof.query_provider();
 
-    let (result, dep_node_index) = start_query(tcx, job_id, query.depth_limit(), || {
-        if query.anon() {
+    let (result, dep_node_index) = start_query(tcx, job_id, query.depth_limit, || {
+        if query.anon {
             // Call the query provider inside an anon task.
-            return dep_graph_data
-                .with_anon_task_inner(tcx, query.dep_kind(), || query.invoke_provider(tcx, key));
+            return dep_graph_data.with_anon_task_inner(tcx, query.dep_kind, || {
+                (query.invoke_provider_fn)(tcx, key)
+            });
         }
 
         // `to_dep_node` is expensive for some `DepKind`s.
@@ -477,8 +479,8 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
             dep_node,
             tcx,
             (query, key),
-            |tcx, (query, key)| query.invoke_provider(tcx, key),
-            query.hash_result(),
+            |tcx, (query, key)| (query.invoke_provider_fn)(tcx, key),
+            query.hash_result,
         )
     });
 
@@ -488,8 +490,8 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 }
 
 #[inline(always)]
-fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     dep_graph_data: &DepGraphData,
     tcx: TyCtxt<'tcx>,
     key: &C::Key,
@@ -526,8 +528,8 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
                 dep_graph_data,
                 &result,
                 prev_dep_node_index,
-                query.hash_result(),
-                query.format_value(),
+                query.hash_result,
+                query.format_value,
             );
         }
 
@@ -555,7 +557,7 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
 
     // The dep-graph for this computation is already in-place.
     // Call the query provider.
-    let result = tcx.dep_graph.with_ignore(|| query.invoke_provider(tcx, *key));
+    let result = tcx.dep_graph.with_ignore(|| (query.invoke_provider_fn)(tcx, *key));
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -573,8 +575,8 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
         dep_graph_data,
         &result,
         prev_dep_node_index,
-        query.hash_result(),
-        query.format_value(),
+        query.hash_result,
+        query.format_value,
     );
 
     Some((result, dep_node_index))
@@ -589,18 +591,18 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+fn ensure_must_run<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: &C::Key,
     check_cache: bool,
 ) -> (bool, Option<DepNode>) {
-    if query.eval_always() {
+    if query.eval_always {
         return (true, None);
     }
 
     // Ensuring an anonymous query makes no sense
-    assert!(!query.anon());
+    assert!(!query.anon);
 
     let dep_node = query.construct_dep_node(tcx, key);
 
@@ -632,20 +634,20 @@ fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
 }
 
 #[inline(always)]
-pub(super) fn get_query_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(super) fn get_query_non_incr<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     span: Span,
     key: C::Key,
 ) -> C::Value {
     debug_assert!(!tcx.dep_graph.is_fully_enabled());
 
-    ensure_sufficient_stack(|| try_execute_query::<C, FLAGS, false>(query, tcx, span, key, None).0)
+    ensure_sufficient_stack(|| try_execute_query::<C, false>(query, tcx, span, key, None).0)
 }
 
 #[inline(always)]
-pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(super) fn get_query_incr<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     span: Span,
     key: C::Key,
@@ -663,17 +665,16 @@ pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
         None
     };
 
-    let (result, dep_node_index) = ensure_sufficient_stack(|| {
-        try_execute_query::<C, FLAGS, true>(query, tcx, span, key, dep_node)
-    });
+    let (result, dep_node_index) =
+        ensure_sufficient_stack(|| try_execute_query::<C, true>(query, tcx, span, key, dep_node));
     if let Some(dep_node_index) = dep_node_index {
         tcx.dep_graph.read_index(dep_node_index)
     }
     Some(result)
 }
 
-pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(crate) fn force_query<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     key: C::Key,
     dep_node: DepNode,
@@ -685,9 +686,9 @@ pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
         return;
     }
 
-    debug_assert!(!query.anon());
+    debug_assert!(!query.anon);
 
     ensure_sufficient_stack(|| {
-        try_execute_query::<C, FLAGS, true>(query, tcx, DUMMY_SP, key, Some(dep_node))
+        try_execute_query::<C, true>(query, tcx, DUMMY_SP, key, Some(dep_node))
     });
 }

compiler/rustc_query_impl/src/lib.rs

@@ -2,28 +2,22 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![feature(adt_const_params)]
 #![feature(core_intrinsics)]
 #![feature(min_specialization)]
 #![feature(rustc_attrs)]
 #![feature(try_blocks)]
 // tidy-alphabetical-end
 
-use std::fmt;
-use std::marker::ConstParamTy;
-
 use rustc_data_structures::sync::AtomicU64;
-use rustc_middle::dep_graph::{self, DepKind, DepNode, DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::dep_graph;
 use rustc_middle::queries::{
     self, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
 };
 use rustc_middle::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
-use rustc_middle::query::plumbing::{
-    HashResult, QueryState, QuerySystem, QuerySystemFns, QueryVTable,
-};
-use rustc_middle::query::{AsLocalKey, CycleError, CycleErrorHandling, QueryCache, QueryMode};
+use rustc_middle::query::plumbing::{QuerySystem, QuerySystemFns, QueryVTable};
+use rustc_middle::query::{AsLocalKey, QueryCache, QueryMode};
 use rustc_middle::ty::TyCtxt;
-use rustc_span::{ErrorGuaranteed, Span};
+use rustc_span::Span;
 
 pub use crate::dep_kind_vtables::make_dep_kind_vtables;
 pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack};
@@ -43,184 +37,9 @@
 mod profiling_support;
 mod values;
 
-#[derive(ConstParamTy)] // Allow this struct to be used for const-generic values.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-struct QueryFlags {
-    /// True if this query has the `anon` modifier.
-    is_anon: bool,
-    /// True if this query has the `depth_limit` modifier.
-    is_depth_limit: bool,
-    /// True if this query has the `feedable` modifier.
-    is_feedable: bool,
-}
-
-/// Combines a [`QueryVTable`] with some additional compile-time booleans.
-/// "Dispatcher" should be understood as a near-synonym of "vtable".
-///
-/// Baking these boolean flags into the type gives a modest but measurable
-/// improvement to compiler perf and compiler code size; see
-/// <https://github.com/rust-lang/rust/pull/151633>.
-struct SemiDynamicQueryDispatcher<'tcx, C: QueryCache, const FLAGS: QueryFlags> {
-    vtable: &'tcx QueryVTable<'tcx, C>,
-}
-
-// Manually implement Copy/Clone, because deriving would put trait bounds on the cache type.
-impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Copy
-    for SemiDynamicQueryDispatcher<'tcx, C, FLAGS>
-{
-}
-impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> Clone
-    for SemiDynamicQueryDispatcher<'tcx, C, FLAGS>
-{
-    fn clone(&self) -> Self {
-        *self
-    }
-}
-
-impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> fmt::Debug
-    for SemiDynamicQueryDispatcher<'tcx, C, FLAGS>
-{
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // When debug-printing a query dispatcher (e.g. for ICE or tracing),
-        // just print the query name to know what query we're dealing with.
-        // The other fields and flags are probably just unhelpful noise.
-        //
-        // If there is need for a more detailed dump of all flags and fields,
-        // consider writing a separate dump method and calling it explicitly.
-        f.write_str(self.name())
-    }
-}
-
-impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'tcx, C, FLAGS> {
-    #[inline(always)]
-    fn name(self) -> &'static str {
-        self.vtable.name
-    }
-
-    #[inline(always)]
-    fn will_cache_on_disk_for_key(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> bool {
-        self.vtable.will_cache_on_disk_for_key_fn.map_or(false, |f| f(tcx, key))
-    }
-
-    // Don't use this method to access query results, instead use the methods on TyCtxt.
-    #[inline(always)]
-    fn query_state(self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> {
-        // Safety:
-        // This is just manually doing the subfield referencing through pointer math.
-        unsafe {
-            &*(&tcx.query_system.states as *const QueryStates<'tcx>)
-                .byte_add(self.vtable.query_state)
-                .cast::<QueryState<'tcx, C::Key>>()
-        }
-    }
-
-    // Don't use this method to access query results, instead use the methods on TyCtxt.
-    #[inline(always)]
-    fn query_cache(self, tcx: TyCtxt<'tcx>) -> &'tcx C {
-        // Safety:
-        // This is just manually doing the subfield referencing through pointer math.
-        unsafe {
-            &*(&tcx.query_system.caches as *const QueryCaches<'tcx>)
-                .byte_add(self.vtable.query_cache)
-                .cast::<C>()
-        }
-    }
-
-    /// Calls `tcx.$query(key)` for this query, and discards the returned value.
-    /// See [`QueryVTable::call_query_method_fn`] for details of this strange operation.
-    #[inline(always)]
-    fn call_query_method(self, tcx: TyCtxt<'tcx>, key: C::Key) {
-        (self.vtable.call_query_method_fn)(tcx, key)
-    }
-
-    /// Calls the actual provider function for this query.
-    /// See [`QueryVTable::invoke_provider_fn`] for more details.
-    #[inline(always)]
-    fn invoke_provider(self, tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value {
-        (self.vtable.invoke_provider_fn)(tcx, key)
-    }
-
-    #[inline(always)]
-    fn try_load_from_disk(
-        self,
-        tcx: TyCtxt<'tcx>,
-        key: &C::Key,
-        prev_index: SerializedDepNodeIndex,
-        index: DepNodeIndex,
-    ) -> Option<C::Value> {
-        // `?` will return None immediately for queries that never cache to disk.
-        self.vtable.try_load_from_disk_fn?(tcx, key, prev_index, index)
-    }
-
-    #[inline]
-    fn is_loadable_from_disk(
-        self,
-        tcx: TyCtxt<'tcx>,
-        key: &C::Key,
-        index: SerializedDepNodeIndex,
-    ) -> bool {
-        self.vtable.is_loadable_from_disk_fn.map_or(false, |f| f(tcx, key, index))
-    }
-
-    /// Synthesize an error value to let compilation continue after a cycle.
-    fn value_from_cycle_error(
-        self,
-        tcx: TyCtxt<'tcx>,
-        cycle_error: &CycleError,
-        guar: ErrorGuaranteed,
-    ) -> C::Value {
-        (self.vtable.value_from_cycle_error)(tcx, cycle_error, guar)
-    }
-
-    #[inline(always)]
-    fn format_value(self) -> fn(&C::Value) -> String {
-        self.vtable.format_value
-    }
-
-    #[inline(always)]
-    fn anon(self) -> bool {
-        FLAGS.is_anon
-    }
-
-    #[inline(always)]
-    fn eval_always(self) -> bool {
-        self.vtable.eval_always
-    }
-
-    #[inline(always)]
-    fn depth_limit(self) -> bool {
-        FLAGS.is_depth_limit
-    }
-
-    #[inline(always)]
-    fn feedable(self) -> bool {
-        FLAGS.is_feedable
-    }
-
-    #[inline(always)]
-    fn dep_kind(self) -> DepKind {
-        self.vtable.dep_kind
-    }
-
-    #[inline(always)]
-    fn cycle_error_handling(self) -> CycleErrorHandling {
-        self.vtable.cycle_error_handling
-    }
-
-    #[inline(always)]
-    fn hash_result(self) -> HashResult<C::Value> {
-        self.vtable.hash_result
-    }
-
-    fn construct_dep_node(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode {
-        DepNode::construct(tcx, self.dep_kind(), key)
-    }
-}
-
-/// Provides access to vtable-like operations for a query
-/// (by creating a [`SemiDynamicQueryDispatcher`]),
-/// but also keeps track of the "unerased" value type of the query
-/// (i.e. the actual result type in the query declaration).
+/// Provides access to vtable-like operations for a query (by obtaining a
+/// `QueryVTable`), but also keeps track of the "unerased" value type of the
+/// query (i.e. the actual result type in the query declaration).
 ///
 /// This trait allows some per-query code to be defined in generic functions
 /// with a trait bound, instead of having to be defined inline within a macro
@@ -228,10 +47,10 @@ fn construct_dep_node(self, tcx: TyCtxt<'tcx>, key: &C::Key) -> DepNode {
 ///
 /// There is one macro-generated implementation of this trait for each query,
 /// on the type `rustc_query_impl::query_impl::$name::QueryType`.
-trait QueryDispatcherUnerased<'tcx, C: QueryCache, const FLAGS: QueryFlags> {
+trait QueryDispatcherUnerased<'tcx, C: QueryCache> {
     type UnerasedValue;
 
-    fn query_dispatcher(tcx: TyCtxt<'tcx>) -> SemiDynamicQueryDispatcher<'tcx, C, FLAGS>;
+    fn query_vtable(tcx: TyCtxt<'tcx>) -> &'tcx QueryVTable<'tcx, C>;
 
     fn restore_val(value: C::Value) -> Self::UnerasedValue;
 }

compiler/rustc_query_impl/src/plumbing.rs

@@ -27,10 +27,10 @@
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::def_id::LOCAL_CRATE;
 
+use crate::QueryDispatcherUnerased;
 use crate::error::{QueryOverflow, QueryOverflowNote};
 use crate::execution::{all_inactive, force_query};
 use crate::job::{QueryJobMap, find_dep_kind_root};
-use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher};
 
 fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) {
     let job_map =
@@ -185,7 +185,7 @@ macro_rules! is_eval_always {
     };
 }
 
-macro_rules! depth_limit {
+macro_rules! is_depth_limit {
     ([]) => {{
         false
     }};
@@ -193,11 +193,11 @@ macro_rules! depth_limit {
         true
     }};
     ([$other:tt $($modifiers:tt)*]) => {
-        depth_limit!([$($modifiers)*])
+        is_depth_limit!([$($modifiers)*])
     };
 }
 
-macro_rules! feedable {
+macro_rules! is_feedable {
     ([]) => {{
         false
     }};
@@ -205,7 +205,7 @@ macro_rules! feedable {
         true
     }};
     ([$other:tt $($modifiers:tt)*]) => {
-        feedable!([$($modifiers)*])
+        is_feedable!([$($modifiers)*])
     };
 }
 
@@ -324,16 +324,16 @@ pub(crate) fn create_deferred_query_stack_frame<'tcx, Cache>(
     QueryStackFrame::new(info, kind, def_id, def_id_for_ty_in_cycle)
 }
 
-pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
 ) where
-    Q: QueryDispatcherUnerased<'tcx, C, FLAGS>,
+    Q: QueryDispatcherUnerased<'tcx, C>,
     Q::UnerasedValue: Encodable<CacheEncoder<'a, 'tcx>>,
 {
-    let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name());
+    let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name);
 
     assert!(all_inactive(query.query_state(tcx)));
     let cache = query.query_cache(tcx);
@@ -351,16 +351,16 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: Quer
     });
 }
 
-pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
 ) {
-    let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name());
+    let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name);
 
     let cache = query.query_cache(tcx);
     let mut map = UnordMap::with_capacity(cache.len());
     cache.iter(&mut |key, _, _| {
-        let node = DepNode::construct(tcx, query.dep_kind(), key);
+        let node = DepNode::construct(tcx, query.dep_kind, key);
         if let Some(other_key) = map.insert(node, *key) {
             bug!(
                 "query key:\n\
@@ -378,8 +378,8 @@ pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags
 }
 
 /// Implementation of [`DepKindVTable::try_load_from_on_disk_cache`] for queries.
-pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     dep_node: DepNode,
 ) {
@@ -394,7 +394,7 @@ pub(crate) fn try_load_from_on_disk_cache_inner<'tcx, C: QueryCache, const FLAGS
     if query.will_cache_on_disk_for_key(tcx, &key) {
         // Call `tcx.$query(key)` for its side-effect of loading the disk-cached
         // value into memory.
-        query.call_query_method(tcx, key);
+        (query.call_query_method_fn)(tcx, key);
     }
 }
 
@@ -431,8 +431,8 @@ pub(crate) fn try_load_from_disk<'tcx, V>(
 }
 
 /// Implementation of [`DepKindVTable::force_from_dep_node`] for queries.
-pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
-    query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
+pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache>(
+    query: &'tcx QueryVTable<'tcx, C>,
     tcx: TyCtxt<'tcx>,
     dep_node: DepNode,
 ) -> bool {
@@ -491,7 +491,7 @@ pub(crate) fn __rust_end_short_backtrace<'tcx>(
                     #[cfg(debug_assertions)]
                     let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
                     execution::get_query_incr(
-                        QueryType::query_dispatcher(tcx),
+                        &tcx.query_system.query_vtables.$name,
                         tcx,
                         span,
                         key,
@@ -511,7 +511,7 @@ pub(crate) fn __rust_end_short_backtrace<'tcx>(
                     __mode: QueryMode,
                 ) -> Option<Erased<queries::$name::Value<'tcx>>> {
                     Some(execution::get_query_non_incr(
-                        QueryType::query_dispatcher(tcx),
+                        &tcx.query_system.query_vtables.$name,
                         tcx,
                         span,
                         key,
@@ -554,7 +554,10 @@ pub(crate) fn make_query_vtable<'tcx>()
             {
                 QueryVTable {
                     name: stringify!($name),
+                    anon: is_anon!([$($modifiers)*]),
                     eval_always: is_eval_always!([$($modifiers)*]),
+                    depth_limit: is_depth_limit!([$($modifiers)*]),
+                    feedable: is_feedable!([$($modifiers)*]),
                     dep_kind: dep_graph::DepKind::$name,
                     cycle_error_handling: cycle_error_handling!([$($modifiers)*]),
                     query_state: std::mem::offset_of!(QueryStates<'tcx>, $name),
@@ -609,24 +612,16 @@ pub(crate) struct QueryType<'tcx> {
                 data: PhantomData<&'tcx ()>
             }
 
-            const FLAGS: QueryFlags = QueryFlags {
-                is_anon: is_anon!([$($modifiers)*]),
-                is_depth_limit: depth_limit!([$($modifiers)*]),
-                is_feedable: feedable!([$($modifiers)*]),
-            };
-
-            impl<'tcx> QueryDispatcherUnerased<'tcx, queries::$name::Storage<'tcx>, FLAGS>
+            impl<'tcx> QueryDispatcherUnerased<'tcx, queries::$name::Storage<'tcx>>
                 for QueryType<'tcx>
             {
                 type UnerasedValue = queries::$name::Value<'tcx>;
 
                 #[inline(always)]
-                fn query_dispatcher(tcx: TyCtxt<'tcx>)
-                    -> SemiDynamicQueryDispatcher<'tcx, queries::$name::Storage<'tcx>, FLAGS>
+                fn query_vtable(tcx: TyCtxt<'tcx>)
+                    -> &'tcx QueryVTable<'tcx, queries::$name::Storage<'tcx>>
                 {
-                    SemiDynamicQueryDispatcher {
-                        vtable: &tcx.query_system.query_vtables.$name,
-                    }
+                    &tcx.query_system.query_vtables.$name
                 }
 
                 #[inline(always)]
@@ -690,10 +685,9 @@ pub(crate) fn encode_query_results<'tcx>(
                 ) {
                     $crate::plumbing::encode_query_results::<
                         query_impl::$name::QueryType<'tcx>,
-                        _,
                         _
                     > (
-                        query_impl::$name::QueryType::query_dispatcher(tcx),
+                        &tcx.query_system.query_vtables.$name,
                         tcx,
                         encoder,
                         query_result_index,
@@ -703,7 +697,7 @@ pub(crate) fn encode_query_results<'tcx>(
 
             pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) {
                 $crate::plumbing::query_key_hash_verify(
-                    query_impl::$name::QueryType::query_dispatcher(tcx),
+                    &tcx.query_system.query_vtables.$name,
                     tcx,
                 )
             }
@@ -780,7 +774,8 @@ mod _dep_kind_vtable_ctors_for_queries {
                 /// `DepKindVTable` constructor for this query.
                 pub(crate) fn $name<'tcx>() -> DepKindVTable<'tcx> {
                     use $crate::query_impl::$name::QueryType;
-                    make_dep_kind_vtable_for_query::<QueryType<'tcx>, _, _>(
+                    make_dep_kind_vtable_for_query::<QueryType<'tcx>, _>(
+                        is_anon!([$($modifiers)*]),
                         is_eval_always!([$($modifiers)*]),
                     )
                 }