Auto merge of #121644 - oli-obk:unique_static_innards2, r=RalfJung,nnethercote

Ensure nested allocations in statics neither get deduplicated nor duplicated

This PR generates new `DefId`s for nested allocations in static items and feeds all the right queries to make the compiler believe these are regular `static` items. I chose this design, because all other designs are fragile and make the compiler horribly complex for such a niche use case.

At present this wrecks incremental compilation performance *in case nested allocations exist* (because any query creating a `DefId` will be recomputed and never loaded from the cache). This will be resolved later in https://github.com/rust-lang/rust/pull/115613 . All other statics are unaffected by this change and will not have performance regressions (heh, famous last words)

This PR contains various smaller refactorings that can be pulled out into separate PRs. It is best reviewed commit-by-commit. The last commit is where the actual magic happens.

r? `@RalfJung` on the const interner and engine changes

fixes https://github.com/rust-lang/rust/issues/79738

7 files changed, 113 insertions, 36 deletions

diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 90884adb28c..5f4408ebbc6 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -37,7 +37,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
             || matches!(
                 ecx.tcx.def_kind(cid.instance.def_id()),
                 DefKind::Const
-                    | DefKind::Static(_)
+                    | DefKind::Static { .. }
                     | DefKind::ConstParam
                     | DefKind::AnonConst
                     | DefKind::InlineConst
@@ -59,7 +59,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
     };
 
     let ret = if let InternKind::Static(_) = intern_kind {
-        create_static_alloc(ecx, cid.instance.def_id(), layout)?
+        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
     } else {
         ecx.allocate(layout, MemoryKind::Stack)?
     };
@@ -380,7 +380,11 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
         }
         Ok(mplace) => {
             // Since evaluation had no errors, validate the resulting constant.
+
+            // Temporarily allow access to the static_root_ids for the purpose of validation.
+            let static_root_ids = ecx.machine.static_root_ids.take();
             let res = const_validate_mplace(&ecx, &mplace, cid);
+            ecx.machine.static_root_ids = static_root_ids;
 
             let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
 
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index f104b836716..dd835279df3 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -8,6 +8,7 @@ use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::fx::IndexEntry;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::LocalDefId;
 use rustc_hir::LangItem;
 use rustc_middle::mir;
 use rustc_middle::mir::AssertMessage;
@@ -59,8 +60,10 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {
     /// Whether to check alignment during evaluation.
     pub(super) check_alignment: CheckAlignment,
 
-    /// Used to prevent reads from a static's base allocation, as that may allow for self-initialization.
-    pub(crate) static_root_alloc_id: Option<AllocId>,
+    /// If `Some`, we are evaluating the initializer of the static with the given `LocalDefId`,
+    /// storing the result in the given `AllocId`.
+    /// Used to prevent reads from a static's base allocation, as that may allow for self-initialization loops.
+    pub(crate) static_root_ids: Option<(AllocId, LocalDefId)>,
 }
 
 #[derive(Copy, Clone)]
@@ -94,7 +97,7 @@ impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
             stack: Vec::new(),
             can_access_mut_global,
             check_alignment,
-            static_root_alloc_id: None,
+            static_root_ids: None,
         }
     }
 }
@@ -749,7 +752,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         ecx: &InterpCx<'mir, 'tcx, Self>,
         alloc_id: AllocId,
     ) -> InterpResult<'tcx> {
-        if Some(alloc_id) == ecx.machine.static_root_alloc_id {
+        if Some(alloc_id) == ecx.machine.static_root_ids.map(|(id, _)| id) {
             Err(ConstEvalErrKind::RecursiveStatic.into())
         } else {
             Ok(())
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 82ce9ecd21d..55d4b9edd59 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -13,12 +13,16 @@
 //! but that would require relying on type information, and given how many ways Rust has to lie
 //! about type information, we want to avoid doing that.
 
+use hir::def::DefKind;
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_middle::mir::interpret::{CtfeProvenance, InterpResult};
+use rustc_middle::mir::interpret::{ConstAllocation, CtfeProvenance, InterpResult};
+use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::TyAndLayout;
+use rustc_span::def_id::LocalDefId;
+use rustc_span::sym;
 
 use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy};
 use crate::const_eval;
@@ -33,7 +37,19 @@ pub trait CompileTimeMachine<'mir, 'tcx: 'mir, T> = Machine<
         FrameExtra = (),
         AllocExtra = (),
         MemoryMap = FxIndexMap<AllocId, (MemoryKind<T>, Allocation)>,
-    >;
+    > + HasStaticRootDefId;
+
+pub trait HasStaticRootDefId {
+    /// Returns the `DefId` of the static item that is currently being evaluated.
+    /// Used for interning to be able to handle nested allocations.
+    fn static_def_id(&self) -> Option<LocalDefId>;
+}
+
+impl HasStaticRootDefId for const_eval::CompileTimeInterpreter<'_, '_> {
+    fn static_def_id(&self) -> Option<LocalDefId> {
+        Some(self.static_root_ids?.1)
+    }
+}
 
 /// Intern an allocation. Returns `Err` if the allocation does not exist in the local memory.
 ///
@@ -67,10 +83,35 @@ fn intern_shallow<'rt, 'mir, 'tcx, T, M: CompileTimeMachine<'mir, 'tcx, T>>(
     }
     // link the alloc id to the actual allocation
     let alloc = ecx.tcx.mk_const_alloc(alloc);
-    ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
+    if let Some(static_id) = ecx.machine.static_def_id() {
+        intern_as_new_static(ecx.tcx, static_id, alloc_id, alloc);
+    } else {
+        ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
+    }
     Ok(alloc.0.0.provenance().ptrs().iter().map(|&(_, prov)| prov))
 }
 
+/// Creates a new `DefId` and feeds all the right queries to make this `DefId`
+/// appear as if it were a user-written `static` (though it has no HIR).
+fn intern_as_new_static<'tcx>(
+    tcx: TyCtxtAt<'tcx>,
+    static_id: LocalDefId,
+    alloc_id: AllocId,
+    alloc: ConstAllocation<'tcx>,
+) {
+    let feed = tcx.create_def(
+        static_id,
+        sym::nested,
+        DefKind::Static { mutability: alloc.0.mutability, nested: true },
+    );
+    tcx.set_nested_alloc_id_static(alloc_id, feed.def_id());
+    feed.codegen_fn_attrs(tcx.codegen_fn_attrs(static_id).clone());
+    feed.eval_static_initializer(Ok(alloc));
+    feed.generics_of(tcx.generics_of(static_id).clone());
+    feed.def_ident_span(tcx.def_ident_span(static_id));
+    feed.explicit_predicates_of(tcx.explicit_predicates_of(static_id));
+}
+
 /// How a constant value should be interned.
 #[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
 pub enum InternKind {
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index cf7f165b87c..e011edcfb29 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -15,6 +15,7 @@ use std::ptr;
 
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
+use rustc_hir::def::DefKind;
 use rustc_middle::mir::display_allocation;
 use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt};
 use rustc_target::abi::{Align, HasDataLayout, Size};
@@ -761,19 +762,36 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // be held throughout the match.
         match self.tcx.try_get_global_alloc(id) {
             Some(GlobalAlloc::Static(def_id)) => {
-                assert!(self.tcx.is_static(def_id));
                 // Thread-local statics do not have a constant address. They *must* be accessed via
                 // `ThreadLocalRef`; we can never have a pointer to them as a regular constant value.
                 assert!(!self.tcx.is_thread_local_static(def_id));
-                // Use size and align of the type.
-                let ty = self
-                    .tcx
-                    .type_of(def_id)
-                    .no_bound_vars()
-                    .expect("statics should not have generic parameters");
-                let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();
-                assert!(layout.is_sized());
-                (layout.size, layout.align.abi, AllocKind::LiveData)
+
+                let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else {
+                    bug!("GlobalAlloc::Static is not a static")
+                };
+
+                let (size, align) = if nested {
+                    // Nested anonymous statics are untyped, so let's get their
+                    // size and alignment from the allocaiton itself. This always
+                    // succeeds, as the query is fed at DefId creation time, so no
+                    // evaluation actually occurs.
+                    let alloc = self.tcx.eval_static_initializer(def_id).unwrap();
+                    (alloc.0.size(), alloc.0.align)
+                } else {
+                    // Use size and align of the type for everything else. We need
+                    // to do that to
+                    // * avoid cycle errors in case of self-referential statics,
+                    // * be able to get information on extern statics.
+                    let ty = self
+                        .tcx
+                        .type_of(def_id)
+                        .no_bound_vars()
+                        .expect("statics should not have generic parameters");
+                    let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();
+                    assert!(layout.is_sized());
+                    (layout.size, layout.align.abi)
+                };
+                (size, align, AllocKind::LiveData)
             }
             Some(GlobalAlloc::Memory(alloc)) => {
                 // Need to duplicate the logic here, because the global allocations have
diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs
index a15e52d07e6..2ed879ca72b 100644
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@@ -22,7 +22,7 @@ pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in
 
 pub use self::eval_context::{format_interp_error, Frame, FrameInfo, InterpCx, StackPopCleanup};
 pub use self::intern::{
-    intern_const_alloc_for_constprop, intern_const_alloc_recursive, InternKind,
+    intern_const_alloc_for_constprop, intern_const_alloc_recursive, HasStaticRootDefId, InternKind,
 };
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs
index 3427368421f..086475f72c5 100644
--- a/compiler/rustc_const_eval/src/interpret/util.rs
+++ b/compiler/rustc_const_eval/src/interpret/util.rs
@@ -1,11 +1,11 @@
 use crate::const_eval::CompileTimeEvalContext;
 use crate::interpret::{MemPlaceMeta, MemoryKind};
+use rustc_hir::def_id::LocalDefId;
 use rustc_middle::mir::interpret::{AllocId, Allocation, InterpResult, Pointer};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{
     self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
-use rustc_span::def_id::DefId;
 use std::ops::ControlFlow;
 
 use super::MPlaceTy;
@@ -89,13 +89,13 @@ pub(crate) fn take_static_root_alloc<'mir, 'tcx: 'mir>(
 
 pub(crate) fn create_static_alloc<'mir, 'tcx: 'mir>(
     ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
-    static_def_id: DefId,
+    static_def_id: LocalDefId,
     layout: TyAndLayout<'tcx>,
 ) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
     let alloc = Allocation::try_uninit(layout.size, layout.align.abi)?;
-    let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id);
-    assert_eq!(ecx.machine.static_root_alloc_id, None);
-    ecx.machine.static_root_alloc_id = Some(alloc_id);
+    let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into());
+    assert_eq!(ecx.machine.static_root_ids, None);
+    ecx.machine.static_root_ids = Some((alloc_id, static_def_id));
     assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none());
     Ok(ecx.ptr_with_meta_to_mplace(Pointer::from(alloc_id).into(), MemPlaceMeta::None, layout))
 }
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 972424bccfa..d18600ce7d7 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -457,15 +457,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                         // Special handling for pointers to statics (irrespective of their type).
                         assert!(!self.ecx.tcx.is_thread_local_static(did));
                         assert!(self.ecx.tcx.is_static(did));
-                        let is_mut =
-                            matches!(self.ecx.tcx.def_kind(did), DefKind::Static(Mutability::Mut))
-                                || !self
-                                    .ecx
-                                    .tcx
-                                    .type_of(did)
-                                    .no_bound_vars()
-                                    .expect("statics should not have generic parameters")
-                                    .is_freeze(*self.ecx.tcx, ty::ParamEnv::reveal_all());
                         // Mode-specific checks
                         match self.ctfe_mode {
                             Some(
@@ -490,8 +481,28 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                             }
                             None => {}
                         }
-                        // Return alloc mutability
-                        if is_mut { Mutability::Mut } else { Mutability::Not }
+                        // Return alloc mutability. For "root" statics we look at the type to account for interior
+                        // mutability; for nested statics we have no type and directly use the annotated mutability.
+                        let DefKind::Static { mutability, nested } = self.ecx.tcx.def_kind(did)
+                        else {
+                            bug!()
+                        };
+                        match (mutability, nested) {
+                            (Mutability::Mut, _) => Mutability::Mut,
+                            (Mutability::Not, true) => Mutability::Not,
+                            (Mutability::Not, false)
+                                if !self
+                                    .ecx
+                                    .tcx
+                                    .type_of(did)
+                                    .no_bound_vars()
+                                    .expect("statics should not have generic parameters")
+                                    .is_freeze(*self.ecx.tcx, ty::ParamEnv::reveal_all()) =>
+                            {
+                                Mutability::Mut
+                            }
+                            (Mutability::Not, false) => Mutability::Not,
+                        }
                     }
                     GlobalAlloc::Memory(alloc) => alloc.inner().mutability,
                     GlobalAlloc::Function(..) | GlobalAlloc::VTable(..) => {