Merge from rust-lang/rust

13 files changed, 854 insertions, 834 deletions

diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index afac08ae6f8..65d586124b3 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -10,7 +10,6 @@
 #![allow(internal_features)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
-#![cfg_attr(not(parallel_compiler), feature(cell_leak))]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
@@ -22,6 +21,7 @@
 #![feature(auto_traits)]
 #![feature(cfg_match)]
 #![feature(core_intrinsics)]
+#![feature(dropck_eyepatch)]
 #![feature(extend_one)]
 #![feature(file_buffered)]
 #![feature(hash_raw_entry)]
@@ -79,6 +79,7 @@ pub mod thinvec;
 pub mod transitive_relation;
 pub mod unhash;
 pub mod unord;
+pub mod vec_cache;
 pub mod work_queue;
 
 mod atomic_ref;
diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs
index 83fdaff515b..2b629024bfe 100644
--- a/compiler/rustc_data_structures/src/marker.rs
+++ b/compiler/rustc_data_structures/src/marker.rs
@@ -1,194 +1,162 @@
-cfg_match! {
-    cfg(not(parallel_compiler)) => {
-        pub auto trait DynSend {}
-        pub auto trait DynSync {}
-
-        impl<T> DynSend for T {}
-        impl<T> DynSync for T {}
-    }
-    _ => {
-        #[rustc_on_unimplemented(
-            message = "`{Self}` doesn't implement `DynSend`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Send`"
-        )]
-        // This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
-        // is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
-        // `Send` type in `IntoDynSyncSend` will create a `DynSend` type.
-        pub unsafe auto trait DynSend {}
-
-        #[rustc_on_unimplemented(
-            message = "`{Self}` doesn't implement `DynSync`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Sync`"
-        )]
-        // This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
-        // is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
-        // `Sync` type in `IntoDynSyncSend` will create a `DynSync` type.
-        pub unsafe auto trait DynSync {}
-
-        // Same with `Sync` and `Send`.
-        unsafe impl<T: DynSync + ?Sized> DynSend for &T {}
-
-        macro_rules! impls_dyn_send_neg {
-            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
-                $(impl$(<$($generics1)*>)? !DynSend for $t1 {})*
-            };
-        }
-
-        // Consistent with `std`
-        impls_dyn_send_neg!(
-            [std::env::Args]
-            [std::env::ArgsOs]
-            [*const T where T: ?Sized]
-            [*mut T where T: ?Sized]
-            [std::ptr::NonNull<T> where T: ?Sized]
-            [std::rc::Rc<T> where T: ?Sized]
-            [std::rc::Weak<T> where T: ?Sized]
-            [std::sync::MutexGuard<'_, T> where T: ?Sized]
-            [std::sync::RwLockReadGuard<'_, T> where T: ?Sized]
-            [std::sync::RwLockWriteGuard<'_, T> where T: ?Sized]
-            [std::io::StdoutLock<'_>]
-            [std::io::StderrLock<'_>]
-        );
-
-        #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
-        // Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
-        impl !DynSend for std::env::VarsOs {}
-
-        macro_rules! already_send {
-            ($([$ty: ty])*) => {
-                $(unsafe impl DynSend for $ty where $ty: Send {})*
-            };
-        }
-
-        // These structures are already `Send`.
-        already_send!(
-            [std::backtrace::Backtrace]
-            [std::io::Stdout]
-            [std::io::Stderr]
-            [std::io::Error]
-            [std::fs::File]
-            [rustc_arena::DroplessArena]
-            [crate::memmap::Mmap]
-            [crate::profiling::SelfProfiler]
-            [crate::owned_slice::OwnedSlice]
-        );
-
-        macro_rules! impl_dyn_send {
-            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
-                $(unsafe impl<$($generics2)*> DynSend for $ty {})*
-            };
-        }
-
-        impl_dyn_send!(
-            [std::sync::atomic::AtomicPtr<T> where T]
-            [std::sync::Mutex<T> where T: ?Sized+ DynSend]
-            [std::sync::mpsc::Sender<T> where T: DynSend]
-            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
-            [std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
-            [std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
-            [std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
-            [std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
-            [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
-            [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
-            [crate::sync::RwLock<T> where T: DynSend]
-            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
-            [rustc_arena::TypedArena<T> where T: DynSend]
-            [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
-            [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
-            [thin_vec::ThinVec<T> where T: DynSend]
-            [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
-        );
-
-        macro_rules! impls_dyn_sync_neg {
-            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
-                $(impl$(<$($generics1)*>)? !DynSync for $t1 {})*
-            };
-        }
-
-        // Consistent with `std`
-        impls_dyn_sync_neg!(
-            [std::env::Args]
-            [std::env::ArgsOs]
-            [*const T where T: ?Sized]
-            [*mut T where T: ?Sized]
-            [std::cell::Cell<T> where T: ?Sized]
-            [std::cell::RefCell<T> where T: ?Sized]
-            [std::cell::UnsafeCell<T> where T: ?Sized]
-            [std::ptr::NonNull<T> where T: ?Sized]
-            [std::rc::Rc<T> where T: ?Sized]
-            [std::rc::Weak<T> where T: ?Sized]
-            [std::cell::OnceCell<T> where T]
-            [std::sync::mpsc::Receiver<T> where T]
-            [std::sync::mpsc::Sender<T> where T]
-        );
-
-        #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
-        // Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
-        impl !DynSync for std::env::VarsOs {}
-
-        macro_rules! already_sync {
-            ($([$ty: ty])*) => {
-                $(unsafe impl DynSync for $ty where $ty: Sync {})*
-            };
-        }
+#[rustc_on_unimplemented(message = "`{Self}` doesn't implement `DynSend`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Send`")]
+// This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
+// is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
+// `Send` type in `IntoDynSyncSend` will create a `DynSend` type.
+pub unsafe auto trait DynSend {}
+
+#[rustc_on_unimplemented(message = "`{Self}` doesn't implement `DynSync`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Sync`")]
+// This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
+// is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
+// `Sync` type in `IntoDynSyncSend` will create a `DynSync` type.
+pub unsafe auto trait DynSync {}
+
+// Same with `Sync` and `Send`.
+unsafe impl<T: DynSync + ?Sized> DynSend for &T {}
+
+macro_rules! impls_dyn_send_neg {
+    ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+        $(impl$(<$($generics1)*>)? !DynSend for $t1 {})*
+    };
+}
 
-        // These structures are already `Sync`.
-        already_sync!(
-            [std::sync::atomic::AtomicBool]
-            [std::sync::atomic::AtomicUsize]
-            [std::sync::atomic::AtomicU8]
-            [std::sync::atomic::AtomicU32]
-            [std::backtrace::Backtrace]
-            [std::io::Error]
-            [std::fs::File]
-            [jobserver_crate::Client]
-            [crate::memmap::Mmap]
-            [crate::profiling::SelfProfiler]
-            [crate::owned_slice::OwnedSlice]
-        );
+// Consistent with `std`
+impls_dyn_send_neg!(
+    [std::env::Args]
+    [std::env::ArgsOs]
+    [*const T where T: ?Sized]
+    [*mut T where T: ?Sized]
+    [std::ptr::NonNull<T> where T: ?Sized]
+    [std::rc::Rc<T> where T: ?Sized]
+    [std::rc::Weak<T> where T: ?Sized]
+    [std::sync::MutexGuard<'_, T> where T: ?Sized]
+    [std::sync::RwLockReadGuard<'_, T> where T: ?Sized]
+    [std::sync::RwLockWriteGuard<'_, T> where T: ?Sized]
+    [std::io::StdoutLock<'_>]
+    [std::io::StderrLock<'_>]
+);
+
+#[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
+// Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
+impl !DynSend for std::env::VarsOs {}
+
+macro_rules! already_send {
+    ($([$ty: ty])*) => {
+        $(unsafe impl DynSend for $ty where $ty: Send {})*
+    };
+}
 
-        // Use portable AtomicU64 for targets without native 64-bit atomics
-        #[cfg(target_has_atomic = "64")]
-        already_sync!(
-            [std::sync::atomic::AtomicU64]
-        );
+// These structures are already `Send`.
+already_send!(
+    [std::backtrace::Backtrace][std::io::Stdout][std::io::Stderr][std::io::Error][std::fs::File]
+        [rustc_arena::DroplessArena][crate::memmap::Mmap][crate::profiling::SelfProfiler]
+        [crate::owned_slice::OwnedSlice]
+);
+
+macro_rules! impl_dyn_send {
+    ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+        $(unsafe impl<$($generics2)*> DynSend for $ty {})*
+    };
+}
 
-        #[cfg(not(target_has_atomic = "64"))]
-        already_sync!(
-            [portable_atomic::AtomicU64]
-        );
+impl_dyn_send!(
+    [std::sync::atomic::AtomicPtr<T> where T]
+    [std::sync::Mutex<T> where T: ?Sized+ DynSend]
+    [std::sync::mpsc::Sender<T> where T: DynSend]
+    [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+    [std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
+    [std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
+    [std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+    [std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
+    [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
+    [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
+    [crate::sync::RwLock<T> where T: DynSend]
+    [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
+    [rustc_arena::TypedArena<T> where T: DynSend]
+    [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
+    [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+    [thin_vec::ThinVec<T> where T: DynSend]
+    [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
+);
+
+macro_rules! impls_dyn_sync_neg {
+    ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+        $(impl$(<$($generics1)*>)? !DynSync for $t1 {})*
+    };
+}
 
-        macro_rules! impl_dyn_sync {
-            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
-                $(unsafe impl<$($generics2)*> DynSync for $ty {})*
-            };
-        }
+// Consistent with `std`
+impls_dyn_sync_neg!(
+    [std::env::Args]
+    [std::env::ArgsOs]
+    [*const T where T: ?Sized]
+    [*mut T where T: ?Sized]
+    [std::cell::Cell<T> where T: ?Sized]
+    [std::cell::RefCell<T> where T: ?Sized]
+    [std::cell::UnsafeCell<T> where T: ?Sized]
+    [std::ptr::NonNull<T> where T: ?Sized]
+    [std::rc::Rc<T> where T: ?Sized]
+    [std::rc::Weak<T> where T: ?Sized]
+    [std::cell::OnceCell<T> where T]
+    [std::sync::mpsc::Receiver<T> where T]
+    [std::sync::mpsc::Sender<T> where T]
+);
+
+#[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
+// Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
+impl !DynSync for std::env::VarsOs {}
+
+macro_rules! already_sync {
+    ($([$ty: ty])*) => {
+        $(unsafe impl DynSync for $ty where $ty: Sync {})*
+    };
+}
 
-        impl_dyn_sync!(
-            [std::sync::atomic::AtomicPtr<T> where T]
-            [std::sync::OnceLock<T> where T: DynSend + DynSync]
-            [std::sync::Mutex<T> where T: ?Sized + DynSend]
-            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
-            [std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
-            [std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
-            [std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
-            [std::collections::BTreeMap<K, V, A> where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync]
-            [Vec<T, A> where T: DynSync, A: std::alloc::Allocator + DynSync]
-            [Box<T, A> where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync]
-            [crate::sync::RwLock<T> where T: DynSend + DynSync]
-            [crate::sync::WorkerLocal<T> where T: DynSend]
-            [crate::intern::Interned<'a, T> where 'a, T: DynSync]
-            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
-            [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
-            [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
-            [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
-            [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
-            [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
-            [thin_vec::ThinVec<T> where T: DynSync]
-        );
-    }
+// These structures are already `Sync`.
+already_sync!(
+    [std::sync::atomic::AtomicBool][std::sync::atomic::AtomicUsize][std::sync::atomic::AtomicU8]
+        [std::sync::atomic::AtomicU32][std::backtrace::Backtrace][std::io::Error][std::fs::File]
+        [jobserver_crate::Client][crate::memmap::Mmap][crate::profiling::SelfProfiler]
+        [crate::owned_slice::OwnedSlice]
+);
+
+// Use portable AtomicU64 for targets without native 64-bit atomics
+#[cfg(target_has_atomic = "64")]
+already_sync!([std::sync::atomic::AtomicU64]);
+
+#[cfg(not(target_has_atomic = "64"))]
+already_sync!([portable_atomic::AtomicU64]);
+
+macro_rules! impl_dyn_sync {
+    ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+        $(unsafe impl<$($generics2)*> DynSync for $ty {})*
+    };
 }
 
+impl_dyn_sync!(
+    [std::sync::atomic::AtomicPtr<T> where T]
+    [std::sync::OnceLock<T> where T: DynSend + DynSync]
+    [std::sync::Mutex<T> where T: ?Sized + DynSend]
+    [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+    [std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
+    [std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
+    [std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+    [std::collections::BTreeMap<K, V, A> where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync]
+    [Vec<T, A> where T: DynSync, A: std::alloc::Allocator + DynSync]
+    [Box<T, A> where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync]
+    [crate::sync::RwLock<T> where T: DynSend + DynSync]
+    [crate::sync::WorkerLocal<T> where T: DynSend]
+    [crate::intern::Interned<'a, T> where 'a, T: DynSync]
+    [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
+    [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
+    [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
+    [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
+    [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+    [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
+    [thin_vec::ThinVec<T> where T: DynSync]
+);
+
 pub fn assert_dyn_sync<T: ?Sized + DynSync>() {}
 pub fn assert_dyn_send<T: ?Sized + DynSend>() {}
 pub fn assert_dyn_send_val<T: ?Sized + DynSend>(_t: &T) {}
@@ -203,7 +171,6 @@ impl<T> FromDyn<T> {
         // Check that `sync::is_dyn_thread_safe()` is true on creation so we can
         // implement `Send` and `Sync` for this structure when `T`
         // implements `DynSend` and `DynSync` respectively.
-        #[cfg(parallel_compiler)]
         assert!(crate::sync::is_dyn_thread_safe());
         FromDyn(val)
     }
@@ -215,11 +182,9 @@ impl<T> FromDyn<T> {
 }
 
 // `FromDyn` is `Send` if `T` is `DynSend`, since it ensures that sync::is_dyn_thread_safe() is true.
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSend> Send for FromDyn<T> {}
 
 // `FromDyn` is `Sync` if `T` is `DynSync`, since it ensures that sync::is_dyn_thread_safe() is true.
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSync> Sync for FromDyn<T> {}
 
 impl<T> std::ops::Deref for FromDyn<T> {
@@ -237,9 +202,7 @@ impl<T> std::ops::Deref for FromDyn<T> {
 #[derive(Copy, Clone)]
 pub struct IntoDynSyncSend<T: ?Sized>(pub T);
 
-#[cfg(parallel_compiler)]
 unsafe impl<T: ?Sized + Send> DynSend for IntoDynSyncSend<T> {}
-#[cfg(parallel_compiler)]
 unsafe impl<T: ?Sized + Sync> DynSync for IntoDynSyncSend<T> {}
 
 impl<T> std::ops::Deref for IntoDynSyncSend<T> {
diff --git a/compiler/rustc_data_structures/src/owned_slice.rs b/compiler/rustc_data_structures/src/owned_slice.rs
index bbe6691e548..c8be0ab52e9 100644
--- a/compiler/rustc_data_structures/src/owned_slice.rs
+++ b/compiler/rustc_data_structures/src/owned_slice.rs
@@ -139,11 +139,9 @@ impl Borrow<[u8]> for OwnedSlice {
 }
 
 // Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Arc<dyn Send + Sync>)`, which is `Send`
-#[cfg(parallel_compiler)]
 unsafe impl sync::Send for OwnedSlice {}
 
 // Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Arc<dyn Send + Sync>)`, which is `Sync`
-#[cfg(parallel_compiler)]
 unsafe impl sync::Sync for OwnedSlice {}
 
 #[cfg(test)]
diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs
index d0b6fe2bc6f..65488c73d3c 100644
--- a/compiler/rustc_data_structures/src/sharded.rs
+++ b/compiler/rustc_data_structures/src/sharded.rs
@@ -3,27 +3,22 @@ use std::collections::hash_map::RawEntryMut;
 use std::hash::{Hash, Hasher};
 use std::{iter, mem};
 
-#[cfg(parallel_compiler)]
 use either::Either;
 
 use crate::fx::{FxHashMap, FxHasher};
-#[cfg(parallel_compiler)]
-use crate::sync::{CacheAligned, is_dyn_thread_safe};
-use crate::sync::{Lock, LockGuard, Mode};
+use crate::sync::{CacheAligned, Lock, LockGuard, Mode, is_dyn_thread_safe};
 
 // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700,
 // but this should be tested on higher core count CPUs. How the `Sharded` type gets used
 // may also affect the ideal number of shards.
 const SHARD_BITS: usize = 5;
 
-#[cfg(parallel_compiler)]
 const SHARDS: usize = 1 << SHARD_BITS;
 
 /// An array of cache-line aligned inner locked structures with convenience methods.
 /// A single field is used when the compiler uses only one thread.
 pub enum Sharded<T> {
     Single(Lock<T>),
-    #[cfg(parallel_compiler)]
     Shards(Box<[CacheAligned<Lock<T>>; SHARDS]>),
 }
 
@@ -37,7 +32,6 @@ impl<T: Default> Default for Sharded<T> {
 impl<T> Sharded<T> {
     #[inline]
     pub fn new(mut value: impl FnMut() -> T) -> Self {
-        #[cfg(parallel_compiler)]
         if is_dyn_thread_safe() {
             return Sharded::Shards(Box::new(
                 [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))),
@@ -52,7 +46,6 @@ impl<T> Sharded<T> {
     pub fn get_shard_by_value<K: Hash + ?Sized>(&self, _val: &K) -> &Lock<T> {
         match self {
             Self::Single(single) => single,
-            #[cfg(parallel_compiler)]
             Self::Shards(..) => self.get_shard_by_hash(make_hash(_val)),
         }
     }
@@ -66,7 +59,6 @@ impl<T> Sharded<T> {
     pub fn get_shard_by_index(&self, _i: usize) -> &Lock<T> {
         match self {
             Self::Single(single) => single,
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
                 // SAFETY: The index gets ANDed with the shard mask, ensuring it is always inbounds.
                 unsafe { &shards.get_unchecked(_i & (SHARDS - 1)).0 }
@@ -87,7 +79,6 @@ impl<T> Sharded<T> {
                 // `might_be_dyn_thread_safe` was also false.
                 unsafe { single.lock_assume(Mode::NoSync) }
             }
-            #[cfg(parallel_compiler)]
             Self::Shards(..) => self.lock_shard_by_hash(make_hash(_val)),
         }
     }
@@ -110,7 +101,6 @@ impl<T> Sharded<T> {
                 // `might_be_dyn_thread_safe` was also false.
                 unsafe { single.lock_assume(Mode::NoSync) }
             }
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
                 // Synchronization is enabled so use the `lock_assume_sync` method optimized
                 // for that case.
@@ -127,11 +117,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn lock_shards(&self) -> impl Iterator<Item = LockGuard<'_, T>> {
         match self {
-            #[cfg(not(parallel_compiler))]
-            Self::Single(single) => iter::once(single.lock()),
-            #[cfg(parallel_compiler)]
             Self::Single(single) => Either::Left(iter::once(single.lock())),
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.lock())),
         }
     }
@@ -139,11 +125,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn try_lock_shards(&self) -> impl Iterator<Item = Option<LockGuard<'_, T>>> {
         match self {
-            #[cfg(not(parallel_compiler))]
-            Self::Single(single) => iter::once(single.try_lock()),
-            #[cfg(parallel_compiler)]
             Self::Single(single) => Either::Left(iter::once(single.try_lock())),
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.try_lock())),
         }
     }
@@ -151,7 +133,6 @@ impl<T> Sharded<T> {
 
 #[inline]
 pub fn shards() -> usize {
-    #[cfg(parallel_compiler)]
     if is_dyn_thread_safe() {
         return SHARDS;
     }
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index a3491dbfec7..7a9533031f4 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -54,9 +54,7 @@ mod worker_local;
 pub use worker_local::{Registry, WorkerLocal};
 
 mod parallel;
-#[cfg(parallel_compiler)]
-pub use parallel::scope;
-pub use parallel::{join, par_for_each_in, par_map, parallel_guard, try_par_for_each_in};
+pub use parallel::{join, par_for_each_in, par_map, parallel_guard, scope, try_par_for_each_in};
 pub use vec::{AppendOnlyIndexVec, AppendOnlyVec};
 
 mod vec;
@@ -104,226 +102,66 @@ mod mode {
     }
 }
 
-pub use mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode};
-
-cfg_match! {
-    cfg(not(parallel_compiler)) => {
-        use std::ops::Add;
-        use std::cell::Cell;
-        use std::sync::atomic::Ordering;
-
-        pub unsafe auto trait Send {}
-        pub unsafe auto trait Sync {}
-
-        unsafe impl<T> Send for T {}
-        unsafe impl<T> Sync for T {}
-
-        /// This is a single threaded variant of `AtomicU64`, `AtomicUsize`, etc.
-        /// It has explicit ordering arguments and is only intended for use with
-        /// the native atomic types.
-        /// You should use this type through the `AtomicU64`, `AtomicUsize`, etc, type aliases
-        /// as it's not intended to be used separately.
-        #[derive(Debug, Default)]
-        pub struct Atomic<T: Copy>(Cell<T>);
-
-        impl<T: Copy> Atomic<T> {
-            #[inline]
-            pub fn new(v: T) -> Self {
-                Atomic(Cell::new(v))
-            }
-
-            #[inline]
-            pub fn into_inner(self) -> T {
-                self.0.into_inner()
-            }
-
-            #[inline]
-            pub fn load(&self, _: Ordering) -> T {
-                self.0.get()
-            }
-
-            #[inline]
-            pub fn store(&self, val: T, _: Ordering) {
-                self.0.set(val)
-            }
-
-            #[inline]
-            pub fn swap(&self, val: T, _: Ordering) -> T {
-                self.0.replace(val)
-            }
-        }
+// FIXME(parallel_compiler): Get rid of these aliases across the compiler.
 
-        impl Atomic<bool> {
-            pub fn fetch_or(&self, val: bool, _: Ordering) -> bool {
-                let old = self.0.get();
-                self.0.set(val | old);
-                old
-            }
-            pub fn fetch_and(&self, val: bool, _: Ordering) -> bool {
-                let old = self.0.get();
-                self.0.set(val & old);
-                old
-            }
-        }
+pub use std::marker::{Send, Sync};
+// Use portable AtomicU64 for targets without native 64-bit atomics
+#[cfg(target_has_atomic = "64")]
+pub use std::sync::atomic::AtomicU64;
+pub use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize};
+pub use std::sync::{Arc as Lrc, OnceLock, Weak};
 
-        impl<T: Copy + PartialEq> Atomic<T> {
-            #[inline]
-            pub fn compare_exchange(&self,
-                                    current: T,
-                                    new: T,
-                                    _: Ordering,
-                                    _: Ordering)
-                                    -> Result<T, T> {
-                let read = self.0.get();
-                if read == current {
-                    self.0.set(new);
-                    Ok(read)
-                } else {
-                    Err(read)
-                }
-            }
-        }
+pub use mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode};
+pub use parking_lot::{
+    MappedMutexGuard as MappedLockGuard, MappedRwLockReadGuard as MappedReadGuard,
+    MappedRwLockWriteGuard as MappedWriteGuard, RwLockReadGuard as ReadGuard,
+    RwLockWriteGuard as WriteGuard,
+};
+#[cfg(not(target_has_atomic = "64"))]
+pub use portable_atomic::AtomicU64;
 
-        impl<T: Add<Output=T> + Copy> Atomic<T> {
-            #[inline]
-            pub fn fetch_add(&self, val: T, _: Ordering) -> T {
-                let old = self.0.get();
-                self.0.set(old + val);
-                old
-            }
-        }
+pub type LRef<'a, T> = &'a T;
 
-        pub type AtomicUsize = Atomic<usize>;
-        pub type AtomicBool = Atomic<bool>;
-        pub type AtomicU32 = Atomic<u32>;
-        pub type AtomicU64 = Atomic<u64>;
-
-        pub use std::rc::Rc as Lrc;
-        pub use std::rc::Weak as Weak;
-        #[doc(no_inline)]
-        pub use std::cell::Ref as ReadGuard;
-        #[doc(no_inline)]
-        pub use std::cell::Ref as MappedReadGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as WriteGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as MappedWriteGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as MappedLockGuard;
-
-        pub use std::cell::OnceCell as OnceLock;
-
-        use std::cell::RefCell as InnerRwLock;
-
-        pub type LRef<'a, T> = &'a mut T;
-
-        #[derive(Debug, Default)]
-        pub struct MTLock<T>(T);
-
-        impl<T> MTLock<T> {
-            #[inline(always)]
-            pub fn new(inner: T) -> Self {
-                MTLock(inner)
-            }
-
-            #[inline(always)]
-            pub fn into_inner(self) -> T {
-                self.0
-            }
-
-            #[inline(always)]
-            pub fn get_mut(&mut self) -> &mut T {
-                &mut self.0
-            }
-
-            #[inline(always)]
-            pub fn lock(&self) -> &T {
-                &self.0
-            }
-
-            #[inline(always)]
-            pub fn lock_mut(&mut self) -> &mut T {
-                &mut self.0
-            }
-        }
+#[derive(Debug, Default)]
+pub struct MTLock<T>(Lock<T>);
 
-        // FIXME: Probably a bad idea (in the threaded case)
-        impl<T: Clone> Clone for MTLock<T> {
-            #[inline]
-            fn clone(&self) -> Self {
-                MTLock(self.0.clone())
-            }
-        }
+impl<T> MTLock<T> {
+    #[inline(always)]
+    pub fn new(inner: T) -> Self {
+        MTLock(Lock::new(inner))
     }
-    _ => {
-        pub use std::marker::Send as Send;
-        pub use std::marker::Sync as Sync;
-
-        pub use parking_lot::RwLockReadGuard as ReadGuard;
-        pub use parking_lot::MappedRwLockReadGuard as MappedReadGuard;
-        pub use parking_lot::RwLockWriteGuard as WriteGuard;
-        pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard;
-
-        pub use parking_lot::MappedMutexGuard as MappedLockGuard;
 
-        pub use std::sync::OnceLock;
-
-        pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32};
-
-        // Use portable AtomicU64 for targets without native 64-bit atomics
-        #[cfg(target_has_atomic = "64")]
-        pub use std::sync::atomic::AtomicU64;
-
-        #[cfg(not(target_has_atomic = "64"))]
-        pub use portable_atomic::AtomicU64;
-
-        pub use std::sync::Arc as Lrc;
-        pub use std::sync::Weak as Weak;
-
-        pub type LRef<'a, T> = &'a T;
-
-        #[derive(Debug, Default)]
-        pub struct MTLock<T>(Lock<T>);
-
-        impl<T> MTLock<T> {
-            #[inline(always)]
-            pub fn new(inner: T) -> Self {
-                MTLock(Lock::new(inner))
-            }
-
-            #[inline(always)]
-            pub fn into_inner(self) -> T {
-                self.0.into_inner()
-            }
-
-            #[inline(always)]
-            pub fn get_mut(&mut self) -> &mut T {
-                self.0.get_mut()
-            }
-
-            #[inline(always)]
-            pub fn lock(&self) -> LockGuard<'_, T> {
-                self.0.lock()
-            }
+    #[inline(always)]
+    pub fn into_inner(self) -> T {
+        self.0.into_inner()
+    }
 
-            #[inline(always)]
-            pub fn lock_mut(&self) -> LockGuard<'_, T> {
-                self.lock()
-            }
-        }
+    #[inline(always)]
+    pub fn get_mut(&mut self) -> &mut T {
+        self.0.get_mut()
+    }
 
-        use parking_lot::RwLock as InnerRwLock;
+    #[inline(always)]
+    pub fn lock(&self) -> LockGuard<'_, T> {
+        self.0.lock()
+    }
 
-        /// This makes locks panic if they are already held.
-        /// It is only useful when you are running in a single thread
-        const ERROR_CHECKING: bool = false;
+    #[inline(always)]
+    pub fn lock_mut(&self) -> LockGuard<'_, T> {
+        self.lock()
     }
 }
 
+use parking_lot::RwLock as InnerRwLock;
+
+/// This makes locks panic if they are already held.
+/// It is only useful when you are running in a single thread
+const ERROR_CHECKING: bool = false;
+
 pub type MTLockRef<'a, T> = LRef<'a, MTLock<T>>;
 
 #[derive(Default)]
-#[cfg_attr(parallel_compiler, repr(align(64)))]
+#[repr(align(64))]
 pub struct CacheAligned<T>(pub T);
 
 pub trait HashMapExt<K, V> {
@@ -357,14 +195,6 @@ impl<T> RwLock<T> {
         self.0.get_mut()
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    #[track_caller]
-    pub fn read(&self) -> ReadGuard<'_, T> {
-        self.0.borrow()
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn read(&self) -> ReadGuard<'_, T> {
         if ERROR_CHECKING {
@@ -380,26 +210,11 @@ impl<T> RwLock<T> {
         f(&*self.read())
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> {
-        self.0.try_borrow_mut().map_err(|_| ())
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> {
         self.0.try_write().ok_or(())
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    #[track_caller]
-    pub fn write(&self) -> WriteGuard<'_, T> {
-        self.0.borrow_mut()
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn write(&self) -> WriteGuard<'_, T> {
         if ERROR_CHECKING {
@@ -427,13 +242,6 @@ impl<T> RwLock<T> {
         self.write()
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    pub fn leak(&self) -> &T {
-        ReadGuard::leak(self.read())
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn leak(&self) -> &T {
         let guard = self.read();
diff --git a/compiler/rustc_data_structures/src/sync/freeze.rs b/compiler/rustc_data_structures/src/sync/freeze.rs
index fad5f583d1c..5236c9fe156 100644
--- a/compiler/rustc_data_structures/src/sync/freeze.rs
+++ b/compiler/rustc_data_structures/src/sync/freeze.rs
@@ -5,9 +5,7 @@ use std::ops::{Deref, DerefMut};
 use std::ptr::NonNull;
 use std::sync::atomic::Ordering;
 
-use crate::sync::{AtomicBool, ReadGuard, RwLock, WriteGuard};
-#[cfg(parallel_compiler)]
-use crate::sync::{DynSend, DynSync};
+use crate::sync::{AtomicBool, DynSend, DynSync, ReadGuard, RwLock, WriteGuard};
 
 /// A type which allows mutation using a lock until
 /// the value is frozen and can be accessed lock-free.
@@ -22,7 +20,6 @@ pub struct FreezeLock<T> {
     lock: RwLock<()>,
 }
 
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSync + DynSend> DynSync for FreezeLock<T> {}
 
 impl<T> FreezeLock<T> {
diff --git a/compiler/rustc_data_structures/src/sync/lock.rs b/compiler/rustc_data_structures/src/sync/lock.rs
index 012ee7f900e..2ccf06ccd4f 100644
--- a/compiler/rustc_data_structures/src/sync/lock.rs
+++ b/compiler/rustc_data_structures/src/sync/lock.rs
@@ -1,236 +1,177 @@
 //! This module implements a lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
 //! It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync` traits.
-//!
-//! When `cfg(parallel_compiler)` is not set, the lock is instead a wrapper around `RefCell`.
 
 #![allow(dead_code)]
 
 use std::fmt;
 
-#[cfg(parallel_compiler)]
-pub use maybe_sync::*;
-#[cfg(not(parallel_compiler))]
-pub use no_sync::*;
-
 #[derive(Clone, Copy, PartialEq)]
 pub enum Mode {
     NoSync,
     Sync,
 }
 
-mod maybe_sync {
-    use std::cell::{Cell, UnsafeCell};
-    use std::intrinsics::unlikely;
-    use std::marker::PhantomData;
-    use std::mem::ManuallyDrop;
-    use std::ops::{Deref, DerefMut};
+use std::cell::{Cell, UnsafeCell};
+use std::intrinsics::unlikely;
+use std::marker::PhantomData;
+use std::mem::ManuallyDrop;
+use std::ops::{Deref, DerefMut};
 
-    use parking_lot::RawMutex;
-    use parking_lot::lock_api::RawMutex as _;
+use parking_lot::RawMutex;
+use parking_lot::lock_api::RawMutex as _;
 
-    use super::Mode;
-    use crate::sync::mode;
-    #[cfg(parallel_compiler)]
-    use crate::sync::{DynSend, DynSync};
+use crate::sync::{DynSend, DynSync, mode};
 
-    /// A guard holding mutable access to a `Lock` which is in a locked state.
-    #[must_use = "if unused the Lock will immediately unlock"]
-    pub struct LockGuard<'a, T> {
-        lock: &'a Lock<T>,
-        marker: PhantomData<&'a mut T>,
+/// A guard holding mutable access to a `Lock` which is in a locked state.
+#[must_use = "if unused the Lock will immediately unlock"]
+pub struct LockGuard<'a, T> {
+    lock: &'a Lock<T>,
+    marker: PhantomData<&'a mut T>,
 
-        /// The synchronization mode of the lock. This is explicitly passed to let LLVM relate it
-        /// to the original lock operation.
-        mode: Mode,
-    }
+    /// The synchronization mode of the lock. This is explicitly passed to let LLVM relate it
+    /// to the original lock operation.
+    mode: Mode,
+}
 
-    impl<'a, T: 'a> Deref for LockGuard<'a, T> {
-        type Target = T;
-        #[inline]
-        fn deref(&self) -> &T {
-            // SAFETY: We have shared access to the mutable access owned by this type,
-            // so we can give out a shared reference.
-            unsafe { &*self.lock.data.get() }
-        }
+impl<'a, T: 'a> Deref for LockGuard<'a, T> {
+    type Target = T;
+    #[inline]
+    fn deref(&self) -> &T {
+        // SAFETY: We have shared access to the mutable access owned by this type,
+        // so we can give out a shared reference.
+        unsafe { &*self.lock.data.get() }
     }
+}
 
-    impl<'a, T: 'a> DerefMut for LockGuard<'a, T> {
-        #[inline]
-        fn deref_mut(&mut self) -> &mut T {
-            // SAFETY: We have mutable access to the data so we can give out a mutable reference.
-            unsafe { &mut *self.lock.data.get() }
-        }
+impl<'a, T: 'a> DerefMut for LockGuard<'a, T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut T {
+        // SAFETY: We have mutable access to the data so we can give out a mutable reference.
+        unsafe { &mut *self.lock.data.get() }
     }
+}
 
-    impl<'a, T: 'a> Drop for LockGuard<'a, T> {
-        #[inline]
-        fn drop(&mut self) {
-            // SAFETY (union access): We get `self.mode` from the lock operation so it is consistent
-            // with the `lock.mode` state. This means we access the right union fields.
-            match self.mode {
-                Mode::NoSync => {
-                    let cell = unsafe { &self.lock.mode_union.no_sync };
-                    debug_assert!(cell.get());
-                    cell.set(false);
-                }
-                // SAFETY (unlock): We know that the lock is locked as this type is a proof of that.
-                Mode::Sync => unsafe { self.lock.mode_union.sync.unlock() },
+impl<'a, T: 'a> Drop for LockGuard<'a, T> {
+    #[inline]
+    fn drop(&mut self) {
+        // SAFETY (union access): We get `self.mode` from the lock operation so it is consistent
+        // with the `lock.mode` state. This means we access the right union fields.
+        match self.mode {
+            Mode::NoSync => {
+                let cell = unsafe { &self.lock.mode_union.no_sync };
+                debug_assert!(cell.get());
+                cell.set(false);
             }
+            // SAFETY (unlock): We know that the lock is locked as this type is a proof of that.
+            Mode::Sync => unsafe { self.lock.mode_union.sync.unlock() },
         }
     }
+}
 
-    union ModeUnion {
-        /// Indicates if the cell is locked. Only used if `Lock.mode` is `NoSync`.
-        no_sync: ManuallyDrop<Cell<bool>>,
+union ModeUnion {
+    /// Indicates if the cell is locked. Only used if `Lock.mode` is `NoSync`.
+    no_sync: ManuallyDrop<Cell<bool>>,
 
-        /// A lock implementation that's only used if `Lock.mode` is `Sync`.
-        sync: ManuallyDrop<RawMutex>,
-    }
+    /// A lock implementation that's only used if `Lock.mode` is `Sync`.
+    sync: ManuallyDrop<RawMutex>,
+}
 
-    /// The value representing a locked state for the `Cell`.
-    const LOCKED: bool = true;
+/// The value representing a locked state for the `Cell`.
+const LOCKED: bool = true;
 
-    /// A lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
-    /// It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync`.
-    pub struct Lock<T> {
-        /// Indicates if synchronization is used via `mode_union.sync` if it's `Sync`, or if a
-        /// not thread safe cell is used via `mode_union.no_sync` if it's `NoSync`.
-        /// This is set on initialization and never changed.
-        mode: Mode,
+/// A lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
+/// It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync`.
+pub struct Lock<T> {
+    /// Indicates if synchronization is used via `mode_union.sync` if it's `Sync`, or if a
+    /// not thread safe cell is used via `mode_union.no_sync` if it's `NoSync`.
+    /// This is set on initialization and never changed.
+    mode: Mode,
 
-        mode_union: ModeUnion,
-        data: UnsafeCell<T>,
-    }
+    mode_union: ModeUnion,
+    data: UnsafeCell<T>,
+}
 
-    impl<T> Lock<T> {
-        #[inline(always)]
-        pub fn new(inner: T) -> Self {
-            let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) {
-                // Create the lock with synchronization enabled using the `RawMutex` type.
-                (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) })
-            } else {
-                // Create the lock with synchronization disabled.
-                (Mode::NoSync, ModeUnion { no_sync: ManuallyDrop::new(Cell::new(!LOCKED)) })
-            };
-            Lock { mode, mode_union, data: UnsafeCell::new(inner) }
-        }
+impl<T> Lock<T> {
+    #[inline(always)]
+    pub fn new(inner: T) -> Self {
+        let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) {
+            // Create the lock with synchronization enabled using the `RawMutex` type.
+            (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) })
+        } else {
+            // Create the lock with synchronization disabled.
+            (Mode::NoSync, ModeUnion { no_sync: ManuallyDrop::new(Cell::new(!LOCKED)) })
+        };
+        Lock { mode, mode_union, data: UnsafeCell::new(inner) }
+    }
 
-        #[inline(always)]
-        pub fn into_inner(self) -> T {
-            self.data.into_inner()
-        }
+    #[inline(always)]
+    pub fn into_inner(self) -> T {
+        self.data.into_inner()
+    }
 
-        #[inline(always)]
-        pub fn get_mut(&mut self) -> &mut T {
-            self.data.get_mut()
-        }
+    #[inline(always)]
+    pub fn get_mut(&mut self) -> &mut T {
+        self.data.get_mut()
+    }
 
-        #[inline(always)]
-        pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
-            let mode = self.mode;
-            // SAFETY: This is safe since the union fields are used in accordance with `self.mode`.
-            match mode {
-                Mode::NoSync => {
-                    let cell = unsafe { &self.mode_union.no_sync };
-                    let was_unlocked = cell.get() != LOCKED;
-                    if was_unlocked {
-                        cell.set(LOCKED);
-                    }
-                    was_unlocked
+    #[inline(always)]
+    pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
+        let mode = self.mode;
+        // SAFETY: This is safe since the union fields are used in accordance with `self.mode`.
+        match mode {
+            Mode::NoSync => {
+                let cell = unsafe { &self.mode_union.no_sync };
+                let was_unlocked = cell.get() != LOCKED;
+                if was_unlocked {
+                    cell.set(LOCKED);
                 }
-                Mode::Sync => unsafe { self.mode_union.sync.try_lock() },
+                was_unlocked
             }
-            .then(|| LockGuard { lock: self, marker: PhantomData, mode })
+            Mode::Sync => unsafe { self.mode_union.sync.try_lock() },
         }
+        .then(|| LockGuard { lock: self, marker: PhantomData, mode })
+    }
 
-        /// This acquires the lock assuming synchronization is in a specific mode.
-        ///
-        /// Safety
-        /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was
-        /// true on lock creation.
-        #[inline(always)]
+    /// This acquires the lock assuming synchronization is in a specific mode.
+    ///
+    /// Safety
+    /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was
+    /// true on lock creation.
+    #[inline(always)]
+    #[track_caller]
+    pub unsafe fn lock_assume(&self, mode: Mode) -> LockGuard<'_, T> {
+        #[inline(never)]
         #[track_caller]
-        pub unsafe fn lock_assume(&self, mode: Mode) -> LockGuard<'_, T> {
-            #[inline(never)]
-            #[track_caller]
-            #[cold]
-            fn lock_held() -> ! {
-                panic!("lock was already held")
-            }
+        #[cold]
+        fn lock_held() -> ! {
+            panic!("lock was already held")
+        }
 
-            // SAFETY: This is safe since the union fields are used in accordance with `mode`
-            // which also must match `self.mode` due to the safety precondition.
-            unsafe {
-                match mode {
-                    Mode::NoSync => {
-                        if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) {
-                            lock_held()
-                        }
+        // SAFETY: This is safe since the union fields are used in accordance with `mode`
+        // which also must match `self.mode` due to the safety precondition.
+        unsafe {
+            match mode {
+                Mode::NoSync => {
+                    if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) {
+                        lock_held()
                     }
-                    Mode::Sync => self.mode_union.sync.lock(),
                 }
+                Mode::Sync => self.mode_union.sync.lock(),
             }
-            LockGuard { lock: self, marker: PhantomData, mode }
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        pub fn lock(&self) -> LockGuard<'_, T> {
-            unsafe { self.lock_assume(self.mode) }
         }
+        LockGuard { lock: self, marker: PhantomData, mode }
     }
 
-    #[cfg(parallel_compiler)]
-    unsafe impl<T: DynSend> DynSend for Lock<T> {}
-    #[cfg(parallel_compiler)]
-    unsafe impl<T: DynSend> DynSync for Lock<T> {}
-}
-
-mod no_sync {
-    use std::cell::RefCell;
-    #[doc(no_inline)]
-    pub use std::cell::RefMut as LockGuard;
-
-    use super::Mode;
-
-    pub struct Lock<T>(RefCell<T>);
-
-    impl<T> Lock<T> {
-        #[inline(always)]
-        pub fn new(inner: T) -> Self {
-            Lock(RefCell::new(inner))
-        }
-
-        #[inline(always)]
-        pub fn into_inner(self) -> T {
-            self.0.into_inner()
-        }
-
-        #[inline(always)]
-        pub fn get_mut(&mut self) -> &mut T {
-            self.0.get_mut()
-        }
-
-        #[inline(always)]
-        pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
-            self.0.try_borrow_mut().ok()
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        // This is unsafe to match the API for the `parallel_compiler` case.
-        pub unsafe fn lock_assume(&self, _mode: Mode) -> LockGuard<'_, T> {
-            self.0.borrow_mut()
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        pub fn lock(&self) -> LockGuard<'_, T> {
-            self.0.borrow_mut()
-        }
+    #[inline(always)]
+    #[track_caller]
+    pub fn lock(&self) -> LockGuard<'_, T> {
+        unsafe { self.lock_assume(self.mode) }
     }
 }
 
+unsafe impl<T: DynSend> DynSend for Lock<T> {}
+unsafe impl<T: DynSend> DynSync for Lock<T> {}
+
 impl<T> Lock<T> {
     #[inline(always)]
     #[track_caller]
diff --git a/compiler/rustc_data_structures/src/sync/parallel.rs b/compiler/rustc_data_structures/src/sync/parallel.rs
index c7df19842d6..1ba631b8623 100644
--- a/compiler/rustc_data_structures/src/sync/parallel.rs
+++ b/compiler/rustc_data_structures/src/sync/parallel.rs
@@ -6,14 +6,11 @@
 use std::any::Any;
 use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
 
-#[cfg(not(parallel_compiler))]
-pub use disabled::*;
-#[cfg(parallel_compiler)]
-pub use enabled::*;
 use parking_lot::Mutex;
+use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
 
 use crate::FatalErrorMarker;
-use crate::sync::IntoDynSyncSend;
+use crate::sync::{DynSend, DynSync, FromDyn, IntoDynSyncSend, mode};
 
 /// A guard used to hold panics that occur during a parallel section to later by unwound.
 /// This is used for the parallel compiler to prevent fatal errors from non-deterministically
@@ -49,65 +46,23 @@ pub fn parallel_guard<R>(f: impl FnOnce(&ParallelGuard) -> R) -> R {
     ret
 }
 
-mod disabled {
-    use crate::sync::parallel_guard;
-
-    #[macro_export]
-    #[cfg(not(parallel_compiler))]
-    macro_rules! parallel {
-        ($($blocks:block),*) => {{
-            $crate::sync::parallel_guard(|guard| {
-                $(guard.run(|| $blocks);)*
-            });
-        }}
-    }
-
-    pub fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
-    where
-        A: FnOnce() -> RA,
-        B: FnOnce() -> RB,
-    {
-        let (a, b) = parallel_guard(|guard| {
-            let a = guard.run(oper_a);
-            let b = guard.run(oper_b);
-            (a, b)
-        });
-        (a.unwrap(), b.unwrap())
-    }
-
-    pub fn par_for_each_in<T: IntoIterator>(t: T, mut for_each: impl FnMut(T::Item)) {
-        parallel_guard(|guard| {
-            t.into_iter().for_each(|i| {
-                guard.run(|| for_each(i));
-            });
-        })
-    }
-
-    pub fn try_par_for_each_in<T: IntoIterator, E>(
-        t: T,
-        mut for_each: impl FnMut(T::Item) -> Result<(), E>,
-    ) -> Result<(), E> {
-        parallel_guard(|guard| {
-            t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
-        })
-    }
-
-    pub fn par_map<T: IntoIterator, R, C: FromIterator<R>>(
-        t: T,
-        mut map: impl FnMut(<<T as IntoIterator>::IntoIter as Iterator>::Item) -> R,
-    ) -> C {
-        parallel_guard(|guard| t.into_iter().filter_map(|i| guard.run(|| map(i))).collect())
-    }
+pub fn serial_join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+where
+    A: FnOnce() -> RA,
+    B: FnOnce() -> RB,
+{
+    let (a, b) = parallel_guard(|guard| {
+        let a = guard.run(oper_a);
+        let b = guard.run(oper_b);
+        (a, b)
+    });
+    (a.unwrap(), b.unwrap())
 }
 
-#[cfg(parallel_compiler)]
-mod enabled {
-    use crate::sync::{DynSend, DynSync, FromDyn, mode, parallel_guard};
-
-    /// Runs a list of blocks in parallel. The first block is executed immediately on
-    /// the current thread. Use that for the longest running block.
-    #[macro_export]
-    macro_rules! parallel {
+/// Runs a list of blocks in parallel. The first block is executed immediately on
+/// the current thread. Use that for the longest running block.
+#[macro_export]
+macro_rules! parallel {
         (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => {
             parallel!(impl $fblock [$block, $($c,)*] [$($rest),*])
         };
@@ -139,92 +94,89 @@ mod enabled {
         };
     }
 
-    // This function only works when `mode::is_dyn_thread_safe()`.
-    pub fn scope<'scope, OP, R>(op: OP) -> R
-    where
-        OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
-        R: DynSend,
-    {
-        let op = FromDyn::from(op);
-        rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
+// This function only works when `mode::is_dyn_thread_safe()`.
+pub fn scope<'scope, OP, R>(op: OP) -> R
+where
+    OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
+    R: DynSend,
+{
+    let op = FromDyn::from(op);
+    rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
+}
+
+#[inline]
+pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
+where
+    A: FnOnce() -> RA + DynSend,
+    B: FnOnce() -> RB + DynSend,
+{
+    if mode::is_dyn_thread_safe() {
+        let oper_a = FromDyn::from(oper_a);
+        let oper_b = FromDyn::from(oper_b);
+        let (a, b) = parallel_guard(|guard| {
+            rayon::join(
+                move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
+                move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
+            )
+        });
+        (a.unwrap().into_inner(), b.unwrap().into_inner())
+    } else {
+        serial_join(oper_a, oper_b)
     }
+}
 
-    #[inline]
-    pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
-    where
-        A: FnOnce() -> RA + DynSend,
-        B: FnOnce() -> RB + DynSend,
-    {
+pub fn par_for_each_in<I, T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>>(
+    t: T,
+    for_each: impl Fn(I) + DynSync + DynSend,
+) {
+    parallel_guard(|guard| {
         if mode::is_dyn_thread_safe() {
-            let oper_a = FromDyn::from(oper_a);
-            let oper_b = FromDyn::from(oper_b);
-            let (a, b) = parallel_guard(|guard| {
-                rayon::join(
-                    move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
-                    move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
-                )
+            let for_each = FromDyn::from(for_each);
+            t.into_par_iter().for_each(|i| {
+                guard.run(|| for_each(i));
             });
-            (a.unwrap().into_inner(), b.unwrap().into_inner())
         } else {
-            super::disabled::join(oper_a, oper_b)
+            t.into_iter().for_each(|i| {
+                guard.run(|| for_each(i));
+            });
         }
-    }
-
-    use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
-
-    pub fn par_for_each_in<I, T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>>(
-        t: T,
-        for_each: impl Fn(I) + DynSync + DynSend,
-    ) {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let for_each = FromDyn::from(for_each);
-                t.into_par_iter().for_each(|i| {
-                    guard.run(|| for_each(i));
-                });
-            } else {
-                t.into_iter().for_each(|i| {
-                    guard.run(|| for_each(i));
-                });
-            }
-        });
-    }
+    });
+}
 
-    pub fn try_par_for_each_in<
-        T: IntoIterator + IntoParallelIterator<Item = <T as IntoIterator>::Item>,
-        E: Send,
-    >(
-        t: T,
-        for_each: impl Fn(<T as IntoIterator>::Item) -> Result<(), E> + DynSync + DynSend,
-    ) -> Result<(), E> {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let for_each = FromDyn::from(for_each);
-                t.into_par_iter()
-                    .filter_map(|i| guard.run(|| for_each(i)))
-                    .reduce(|| Ok(()), Result::and)
-            } else {
-                t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
-            }
-        })
-    }
+pub fn try_par_for_each_in<
+    T: IntoIterator + IntoParallelIterator<Item = <T as IntoIterator>::Item>,
+    E: Send,
+>(
+    t: T,
+    for_each: impl Fn(<T as IntoIterator>::Item) -> Result<(), E> + DynSync + DynSend,
+) -> Result<(), E> {
+    parallel_guard(|guard| {
+        if mode::is_dyn_thread_safe() {
+            let for_each = FromDyn::from(for_each);
+            t.into_par_iter()
+                .filter_map(|i| guard.run(|| for_each(i)))
+                .reduce(|| Ok(()), Result::and)
+        } else {
+            t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
+        }
+    })
+}
 
-    pub fn par_map<
-        I,
-        T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>,
-        R: std::marker::Send,
-        C: FromIterator<R> + FromParallelIterator<R>,
-    >(
-        t: T,
-        map: impl Fn(I) -> R + DynSync + DynSend,
-    ) -> C {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let map = FromDyn::from(map);
-                t.into_par_iter().filter_map(|i| guard.run(|| map(i))).collect()
-            } else {
-                t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()
-            }
-        })
-    }
+pub fn par_map<
+    I,
+    T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>,
+    R: std::marker::Send,
+    C: FromIterator<R> + FromParallelIterator<R>,
+>(
+    t: T,
+    map: impl Fn(I) -> R + DynSync + DynSend,
+) -> C {
+    parallel_guard(|guard| {
+        if mode::is_dyn_thread_safe() {
+            let map = FromDyn::from(map);
+            t.into_par_iter().filter_map(|i| guard.run(|| map(i))).collect()
+        } else {
+            t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()
+        }
+    })
 }
diff --git a/compiler/rustc_data_structures/src/sync/vec.rs b/compiler/rustc_data_structures/src/sync/vec.rs
index 314496ce9f0..21ec5cf6c13 100644
--- a/compiler/rustc_data_structures/src/sync/vec.rs
+++ b/compiler/rustc_data_structures/src/sync/vec.rs
@@ -4,40 +4,23 @@ use rustc_index::Idx;
 
 #[derive(Default)]
 pub struct AppendOnlyIndexVec<I: Idx, T: Copy> {
-    #[cfg(not(parallel_compiler))]
-    vec: elsa::vec::FrozenVec<T>,
-    #[cfg(parallel_compiler)]
     vec: elsa::sync::LockFreeFrozenVec<T>,
     _marker: PhantomData<fn(&I)>,
 }
 
 impl<I: Idx, T: Copy> AppendOnlyIndexVec<I, T> {
     pub fn new() -> Self {
-        Self {
-            #[cfg(not(parallel_compiler))]
-            vec: elsa::vec::FrozenVec::new(),
-            #[cfg(parallel_compiler)]
-            vec: elsa::sync::LockFreeFrozenVec::new(),
-            _marker: PhantomData,
-        }
+        Self { vec: elsa::sync::LockFreeFrozenVec::new(), _marker: PhantomData }
     }
 
     pub fn push(&self, val: T) -> I {
-        #[cfg(not(parallel_compiler))]
-        let i = self.vec.len();
-        #[cfg(not(parallel_compiler))]
-        self.vec.push(val);
-        #[cfg(parallel_compiler)]
         let i = self.vec.push(val);
         I::new(i)
     }
 
     pub fn get(&self, i: I) -> Option<T> {
         let i = i.index();
-        #[cfg(not(parallel_compiler))]
-        return self.vec.get_copy(i);
-        #[cfg(parallel_compiler)]
-        return self.vec.get(i);
+        self.vec.get(i)
     }
 }
 
diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs
index b6efcada10b..d75af009850 100644
--- a/compiler/rustc_data_structures/src/sync/worker_local.rs
+++ b/compiler/rustc_data_structures/src/sync/worker_local.rs
@@ -5,8 +5,9 @@ use std::ptr;
 use std::sync::Arc;
 
 use parking_lot::Mutex;
-#[cfg(parallel_compiler)]
-use {crate::outline, crate::sync::CacheAligned};
+
+use crate::outline;
+use crate::sync::CacheAligned;
 
 /// A pointer to the `RegistryData` which uniquely identifies a registry.
 /// This identifier can be reused if the registry gets freed.
@@ -21,7 +22,6 @@ impl RegistryId {
     ///
     /// Note that there's a race possible where the identifier in `THREAD_DATA` could be reused
     /// so this can succeed from a different registry.
-    #[cfg(parallel_compiler)]
     fn verify(self) -> usize {
         let (id, index) = THREAD_DATA.with(|data| (data.registry_id.get(), data.index.get()));
 
@@ -102,11 +102,7 @@ impl Registry {
 /// worker local value through the `Deref` impl on the registry associated with the thread it was
 /// created on. It will panic otherwise.
 pub struct WorkerLocal<T> {
-    #[cfg(not(parallel_compiler))]
-    local: T,
-    #[cfg(parallel_compiler)]
     locals: Box<[CacheAligned<T>]>,
-    #[cfg(parallel_compiler)]
     registry: Registry,
 }
 
@@ -114,7 +110,6 @@ pub struct WorkerLocal<T> {
 // or it will panic for threads without an associated local. So there isn't a need for `T` to do
 // it's own synchronization. The `verify` method on `RegistryId` has an issue where the id
 // can be reused, but `WorkerLocal` has a reference to `Registry` which will prevent any reuse.
-#[cfg(parallel_compiler)]
 unsafe impl<T: Send> Sync for WorkerLocal<T> {}
 
 impl<T> WorkerLocal<T> {
@@ -122,33 +117,17 @@ impl<T> WorkerLocal<T> {
     /// value this worker local should take for each thread in the registry.
     #[inline]
     pub fn new<F: FnMut(usize) -> T>(mut initial: F) -> WorkerLocal<T> {
-        #[cfg(parallel_compiler)]
-        {
-            let registry = Registry::current();
-            WorkerLocal {
-                locals: (0..registry.0.thread_limit.get())
-                    .map(|i| CacheAligned(initial(i)))
-                    .collect(),
-                registry,
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            WorkerLocal { local: initial(0) }
+        let registry = Registry::current();
+        WorkerLocal {
+            locals: (0..registry.0.thread_limit.get()).map(|i| CacheAligned(initial(i))).collect(),
+            registry,
         }
     }
 
     /// Returns the worker-local values for each thread
     #[inline]
     pub fn into_inner(self) -> impl Iterator<Item = T> {
-        #[cfg(parallel_compiler)]
-        {
-            self.locals.into_vec().into_iter().map(|local| local.0)
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            std::iter::once(self.local)
-        }
+        self.locals.into_vec().into_iter().map(|local| local.0)
     }
 }
 
@@ -156,13 +135,6 @@ impl<T> Deref for WorkerLocal<T> {
     type Target = T;
 
     #[inline(always)]
-    #[cfg(not(parallel_compiler))]
-    fn deref(&self) -> &T {
-        &self.local
-    }
-
-    #[inline(always)]
-    #[cfg(parallel_compiler)]
     fn deref(&self) -> &T {
         // This is safe because `verify` will only return values less than
         // `self.registry.thread_limit` which is the size of the `self.locals` array.
diff --git a/compiler/rustc_data_structures/src/unord.rs b/compiler/rustc_data_structures/src/unord.rs
index bafb16a8b5e..34895d3efe6 100644
--- a/compiler/rustc_data_structures/src/unord.rs
+++ b/compiler/rustc_data_structures/src/unord.rs
@@ -602,6 +602,11 @@ impl<K: Eq + Hash, V> UnordMap<K, V> {
             .into_iter()
             .map(|(_, v)| v)
     }
+
+    #[inline]
+    pub fn clear(&mut self) {
+        self.inner.clear()
+    }
 }
 
 impl<K, Q: ?Sized, V> Index<&Q> for UnordMap<K, V>
diff --git a/compiler/rustc_data_structures/src/vec_cache.rs b/compiler/rustc_data_structures/src/vec_cache.rs
new file mode 100644
index 00000000000..eb251b587c8
--- /dev/null
+++ b/compiler/rustc_data_structures/src/vec_cache.rs
@@ -0,0 +1,324 @@
+//! VecCache maintains a mapping from K -> (V, I) pairing. K and I must be roughly u32-sized, and V
+//! must be Copy.
+//!
+//! VecCache supports efficient concurrent put/get across the key space, with write-once semantics
+//! (i.e., a given key can only be put once). Subsequent puts will panic.
+//!
+//! This is currently used for query caching.
+
+use std::fmt::Debug;
+use std::marker::PhantomData;
+use std::sync::atomic::{AtomicPtr, AtomicU32, AtomicUsize, Ordering};
+
+use rustc_index::Idx;
+
+struct Slot<V> {
+    // We never construct &Slot<V> so it's fine for this to not be in an UnsafeCell.
+    value: V,
+    // This is both an index and a once-lock.
+    //
+    // 0: not yet initialized.
+    // 1: lock held, initializing.
+    // 2..u32::MAX - 2: initialized.
+    index_and_lock: AtomicU32,
+}
+
+/// This uniquely identifies a single `Slot<V>` entry in the buckets map, and provides accessors for
+/// either getting the value or putting a value.
+#[derive(Copy, Clone, Debug)]
+struct SlotIndex {
+    // the index of the bucket in VecCache (0 to 20)
+    bucket_idx: usize,
+    // number of entries in that bucket
+    entries: usize,
+    // the index of the slot within the bucket
+    index_in_bucket: usize,
+}
+
+// This makes sure the counts are consistent with what we allocate, precomputing each bucket a
+// compile-time. Visiting all powers of two is enough to hit all the buckets.
+//
+// We confirm counts are accurate in the slot_index_exhaustive test.
+const ENTRIES_BY_BUCKET: [usize; 21] = {
+    let mut entries = [0; 21];
+    let mut key = 0;
+    loop {
+        let si = SlotIndex::from_index(key);
+        entries[si.bucket_idx] = si.entries;
+        if key == 0 {
+            key = 1;
+        } else if key == (1 << 31) {
+            break;
+        } else {
+            key <<= 1;
+        }
+    }
+    entries
+};
+
+impl SlotIndex {
+    // This unpacks a flat u32 index into identifying which bucket it belongs to and the offset
+    // within that bucket. As noted in the VecCache docs, buckets double in size with each index.
+    // Typically that would mean 31 buckets (2^0 + 2^1 ... + 2^31 = u32::MAX - 1), but to reduce
+    // the size of the VecCache struct and avoid uselessly small allocations, we instead have the
+    // first bucket have 2**12 entries. To simplify the math, the second bucket also 2**12 entries,
+    // and buckets double from there.
+    //
+    // We assert that [0, 2**32 - 1] uniquely map through this function to individual, consecutive
+    // slots (see `slot_index_exhaustive` in tests).
+    #[inline]
+    const fn from_index(idx: u32) -> Self {
+        let mut bucket = match idx.checked_ilog2() {
+            Some(x) => x as usize,
+            None => 0,
+        };
+        let entries;
+        let running_sum;
+        if bucket <= 11 {
+            entries = 1 << 12;
+            running_sum = 0;
+            bucket = 0;
+        } else {
+            entries = 1 << bucket;
+            running_sum = entries;
+            bucket = bucket - 11;
+        }
+        SlotIndex { bucket_idx: bucket, entries, index_in_bucket: idx as usize - running_sum }
+    }
+
+    // SAFETY: Buckets must be managed solely by functions here (i.e., get/put on SlotIndex) and
+    // `self` comes from SlotIndex::from_index
+    #[inline]
+    unsafe fn get<V: Copy>(&self, buckets: &[AtomicPtr<Slot<V>>; 21]) -> Option<(V, u32)> {
+        // SAFETY: `bucket_idx` is ilog2(u32).saturating_sub(11), which is at most 21, i.e.,
+        // in-bounds of buckets. See `from_index` for computation.
+        let bucket = unsafe { buckets.get_unchecked(self.bucket_idx) };
+        let ptr = bucket.load(Ordering::Acquire);
+        // Bucket is not yet initialized: then we obviously won't find this entry in that bucket.
+        if ptr.is_null() {
+            return None;
+        }
+        assert!(self.index_in_bucket < self.entries);
+        // SAFETY: `bucket` was allocated (so <= isize in total bytes) to hold `entries`, so this
+        // must be inbounds.
+        let slot = unsafe { ptr.add(self.index_in_bucket) };
+
+        // SAFETY: initialized bucket has zeroed all memory within the bucket, so we are valid for
+        // AtomicU32 access.
+        let index_and_lock = unsafe { &(*slot).index_and_lock };
+        let current = index_and_lock.load(Ordering::Acquire);
+        let index = match current {
+            0 => return None,
+            // Treat "initializing" as actually just not initialized at all.
+            // The only reason this is a separate state is that `complete` calls could race and
+            // we can't allow that, but from load perspective there's no difference.
+            1 => return None,
+            _ => current - 2,
+        };
+
+        // SAFETY:
+        // * slot is a valid pointer (buckets are always valid for the index we get).
+        // * value is initialized since we saw a >= 2 index above.
+        // * `V: Copy`, so safe to read.
+        let value = unsafe { (*slot).value };
+        Some((value, index))
+    }
+
+    fn bucket_ptr<V>(&self, bucket: &AtomicPtr<Slot<V>>) -> *mut Slot<V> {
+        let ptr = bucket.load(Ordering::Acquire);
+        if ptr.is_null() { self.initialize_bucket(bucket) } else { ptr }
+    }
+
+    #[cold]
+    fn initialize_bucket<V>(&self, bucket: &AtomicPtr<Slot<V>>) -> *mut Slot<V> {
+        static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
+        // If we are initializing the bucket, then acquire a global lock.
+        //
+        // This path is quite cold, so it's cheap to use a global lock. This ensures that we never
+        // have multiple allocations for the same bucket.
+        let _allocator_guard = LOCK.lock().unwrap_or_else(|e| e.into_inner());
+
+        let ptr = bucket.load(Ordering::Acquire);
+
+        // OK, now under the allocator lock, if we're still null then it's definitely us that will
+        // initialize this bucket.
+        if ptr.is_null() {
+            let bucket_layout =
+                std::alloc::Layout::array::<Slot<V>>(self.entries as usize).unwrap();
+            // This is more of a sanity check -- this code is very cold, so it's safe to pay a
+            // little extra cost here.
+            assert!(bucket_layout.size() > 0);
+            // SAFETY: Just checked that size is non-zero.
+            let allocated = unsafe { std::alloc::alloc_zeroed(bucket_layout).cast::<Slot<V>>() };
+            if allocated.is_null() {
+                std::alloc::handle_alloc_error(bucket_layout);
+            }
+            bucket.store(allocated, Ordering::Release);
+            allocated
+        } else {
+            // Otherwise some other thread initialized this bucket after we took the lock. In that
+            // case, just return early.
+            ptr
+        }
+    }
+
+    /// Returns true if this successfully put into the map.
+    #[inline]
+    fn put<V>(&self, buckets: &[AtomicPtr<Slot<V>>; 21], value: V, extra: u32) -> bool {
+        // SAFETY: `bucket_idx` is ilog2(u32).saturating_sub(11), which is at most 21, i.e.,
+        // in-bounds of buckets.
+        let bucket = unsafe { buckets.get_unchecked(self.bucket_idx) };
+        let ptr = self.bucket_ptr(bucket);
+
+        assert!(self.index_in_bucket < self.entries);
+        // SAFETY: `bucket` was allocated (so <= isize in total bytes) to hold `entries`, so this
+        // must be inbounds.
+        let slot = unsafe { ptr.add(self.index_in_bucket) };
+
+        // SAFETY: initialized bucket has zeroed all memory within the bucket, so we are valid for
+        // AtomicU32 access.
+        let index_and_lock = unsafe { &(*slot).index_and_lock };
+        match index_and_lock.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) {
+            Ok(_) => {
+                // We have acquired the initialization lock. It is our job to write `value` and
+                // then set the lock to the real index.
+
+                unsafe {
+                    (&raw mut (*slot).value).write(value);
+                }
+
+                index_and_lock.store(extra.checked_add(2).unwrap(), Ordering::Release);
+
+                true
+            }
+
+            // Treat "initializing" as the caller's fault. Callers are responsible for ensuring that
+            // there are no races on initialization. In the compiler's current usage for query
+            // caches, that's the "active query map" which ensures each query actually runs once
+            // (even if concurrently started).
+            Err(1) => panic!("caller raced calls to put()"),
+
+            // This slot was already populated. Also ignore, currently this is the same as
+            // "initializing".
+            Err(_) => false,
+        }
+    }
+}
+
+pub struct VecCache<K: Idx, V, I> {
+    // Entries per bucket:
+    // Bucket  0:       4096 2^12
+    // Bucket  1:       4096 2^12
+    // Bucket  2:       8192
+    // Bucket  3:      16384
+    // ...
+    // Bucket 19: 1073741824
+    // Bucket 20: 2147483648
+    // The total number of entries if all buckets are initialized is u32::MAX-1.
+    buckets: [AtomicPtr<Slot<V>>; 21],
+
+    // In the compiler's current usage these are only *read* during incremental and self-profiling.
+    // They are an optimization over iterating the full buckets array.
+    present: [AtomicPtr<Slot<()>>; 21],
+    len: AtomicUsize,
+
+    key: PhantomData<(K, I)>,
+}
+
+impl<K: Idx, V, I> Default for VecCache<K, V, I> {
+    fn default() -> Self {
+        VecCache {
+            buckets: Default::default(),
+            key: PhantomData,
+            len: Default::default(),
+            present: Default::default(),
+        }
+    }
+}
+
+// SAFETY: No access to `V` is made.
+unsafe impl<K: Idx, #[may_dangle] V, I> Drop for VecCache<K, V, I> {
+    fn drop(&mut self) {
+        // We have unique ownership, so no locks etc. are needed. Since `K` and `V` are both `Copy`,
+        // we are also guaranteed to just need to deallocate any large arrays (not iterate over
+        // contents).
+        //
+        // Confirm no need to deallocate invidual entries. Note that `V: Copy` is asserted on
+        // insert/lookup but not necessarily construction, primarily to avoid annoyingly propagating
+        // the bounds into struct definitions everywhere.
+        assert!(!std::mem::needs_drop::<K>());
+        assert!(!std::mem::needs_drop::<V>());
+
+        for (idx, bucket) in self.buckets.iter().enumerate() {
+            let bucket = bucket.load(Ordering::Acquire);
+            if !bucket.is_null() {
+                let layout = std::alloc::Layout::array::<Slot<V>>(ENTRIES_BY_BUCKET[idx]).unwrap();
+                unsafe {
+                    std::alloc::dealloc(bucket.cast(), layout);
+                }
+            }
+        }
+
+        for (idx, bucket) in self.present.iter().enumerate() {
+            let bucket = bucket.load(Ordering::Acquire);
+            if !bucket.is_null() {
+                let layout = std::alloc::Layout::array::<Slot<()>>(ENTRIES_BY_BUCKET[idx]).unwrap();
+                unsafe {
+                    std::alloc::dealloc(bucket.cast(), layout);
+                }
+            }
+        }
+    }
+}
+
+impl<K, V, I> VecCache<K, V, I>
+where
+    K: Eq + Idx + Copy + Debug,
+    V: Copy,
+    I: Idx + Copy,
+{
+    #[inline(always)]
+    pub fn lookup(&self, key: &K) -> Option<(V, I)> {
+        let key = u32::try_from(key.index()).unwrap();
+        let slot_idx = SlotIndex::from_index(key);
+        match unsafe { slot_idx.get(&self.buckets) } {
+            Some((value, idx)) => Some((value, I::new(idx as usize))),
+            None => None,
+        }
+    }
+
+    #[inline]
+    pub fn complete(&self, key: K, value: V, index: I) {
+        let key = u32::try_from(key.index()).unwrap();
+        let slot_idx = SlotIndex::from_index(key);
+        if slot_idx.put(&self.buckets, value, index.index() as u32) {
+            let present_idx = self.len.fetch_add(1, Ordering::Relaxed);
+            let slot = SlotIndex::from_index(present_idx as u32);
+            // We should always be uniquely putting due to `len` fetch_add returning unique values.
+            assert!(slot.put(&self.present, (), key));
+        }
+    }
+
+    pub fn iter(&self, f: &mut dyn FnMut(&K, &V, I)) {
+        for idx in 0..self.len.load(Ordering::Acquire) {
+            let key = SlotIndex::from_index(idx as u32);
+            match unsafe { key.get(&self.present) } {
+                // This shouldn't happen in our current usage (iter is really only
+                // used long after queries are done running), but if we hit this in practice it's
+                // probably fine to just break early.
+                None => unreachable!(),
+                Some(((), key)) => {
+                    let key = K::new(key as usize);
+                    // unwrap() is OK: present entries are always written only after we put the real
+                    // entry.
+                    let value = self.lookup(&key).unwrap();
+                    f(&key, &value.0, value.1);
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/compiler/rustc_data_structures/src/vec_cache/tests.rs b/compiler/rustc_data_structures/src/vec_cache/tests.rs
new file mode 100644
index 00000000000..a05f2741362
--- /dev/null
+++ b/compiler/rustc_data_structures/src/vec_cache/tests.rs
@@ -0,0 +1,95 @@
+use super::*;
+
+#[test]
+#[cfg(not(miri))]
+fn vec_cache_empty() {
+    let cache: VecCache<u32, u32, u32> = VecCache::default();
+    for key in 0..u32::MAX {
+        assert!(cache.lookup(&key).is_none());
+    }
+}
+
+#[test]
+fn vec_cache_insert_and_check() {
+    let cache: VecCache<u32, u32, u32> = VecCache::default();
+    cache.complete(0, 1, 2);
+    assert_eq!(cache.lookup(&0), Some((1, 2)));
+}
+
+#[test]
+fn sparse_inserts() {
+    let cache: VecCache<u32, u8, u32> = VecCache::default();
+    let end = if cfg!(target_pointer_width = "64") && cfg!(target_os = "linux") {
+        // For paged memory, 64-bit systems we should be able to sparsely allocate all of the pages
+        // needed for these inserts cheaply (without needing to actually have gigabytes of resident
+        // memory).
+        31
+    } else {
+        // Otherwise, still run the test but scaled back:
+        //
+        // Each slot is 5 bytes, so 2^25 entries (on non-virtual memory systems, like e.g. Windows) will
+        // mean 160 megabytes of allocated memory. Going beyond that is probably not reasonable for
+        // tests.
+        25
+    };
+    for shift in 0..end {
+        let key = 1u32 << shift;
+        cache.complete(key, shift, key);
+        assert_eq!(cache.lookup(&key), Some((shift, key)));
+    }
+}
+
+#[test]
+fn concurrent_stress_check() {
+    let cache: VecCache<u32, u32, u32> = VecCache::default();
+    std::thread::scope(|s| {
+        for idx in 0..100 {
+            let cache = &cache;
+            s.spawn(move || {
+                cache.complete(idx, idx, idx);
+            });
+        }
+    });
+
+    for idx in 0..100 {
+        assert_eq!(cache.lookup(&idx), Some((idx, idx)));
+    }
+}
+
+#[test]
+fn slot_entries_table() {
+    assert_eq!(ENTRIES_BY_BUCKET, [
+        4096, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304,
+        8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824,
+        2147483648
+    ]);
+}
+
+#[test]
+#[cfg(not(miri))]
+fn slot_index_exhaustive() {
+    let mut buckets = [0u32; 21];
+    for idx in 0..=u32::MAX {
+        buckets[SlotIndex::from_index(idx).bucket_idx] += 1;
+    }
+    let mut prev = None::<SlotIndex>;
+    for idx in 0..=u32::MAX {
+        let slot_idx = SlotIndex::from_index(idx);
+        if let Some(p) = prev {
+            if p.bucket_idx == slot_idx.bucket_idx {
+                assert_eq!(p.index_in_bucket + 1, slot_idx.index_in_bucket);
+            } else {
+                assert_eq!(slot_idx.index_in_bucket, 0);
+            }
+        } else {
+            assert_eq!(idx, 0);
+            assert_eq!(slot_idx.index_in_bucket, 0);
+            assert_eq!(slot_idx.bucket_idx, 0);
+        }
+
+        assert_eq!(buckets[slot_idx.bucket_idx], slot_idx.entries as u32);
+        assert_eq!(ENTRIES_BY_BUCKET[slot_idx.bucket_idx], slot_idx.entries, "{}", idx);
+
+        prev = Some(slot_idx);
+    }
+}