std: refactor the TLS implementation

As discovered by Mara in #110897, our TLS implementation is a total mess. In the past months, I have simplified the actual macros and their expansions, but the majority of the complexity comes from the platform-specific support code needed to create keys and register destructors. In keeping with #117276, I have therefore moved all of the `thread_local_key`/`thread_local_dtor` modules to the `thread_local` module in `sys` and merged them into a new structure, so that future porters of `std` can simply mix-and-match the existing code instead of having to copy the same (bad) implementation everywhere. The new structure should become obvious when looking at `sys/thread_local/mod.rs`.

Unfortunately, the documentation changes associated with the refactoring have made this PR rather large. That said, this contains no functional changes except for two small ones:
* the key-based destructor fallback now, by virtue of sharing the implementation used by macOS and others, stores its list in a `#[thread_local]` static instead of in the key, eliminating one indirection layer and drastically simplifying its code.
* I've switched over ZKVM (tier 3) to use the same implementation as WebAssembly, as the implementation was just a way worse version of that

Please let me know if I can make this easier to review! I know these large PRs aren't optimal, but I couldn't think of any good intermediate steps.

@rustbot label +A-thread-locals

18 files changed, 1065 insertions, 26 deletions

diff --git a/library/std/src/sys/thread_local/destructors/linux.rs b/library/std/src/sys/thread_local/destructors/linux.rs
new file mode 100644
index 00000000000..e00ca9a91bd
--- /dev/null
+++ b/library/std/src/sys/thread_local/destructors/linux.rs
@@ -0,0 +1,58 @@
+//! Destructor registration for Linux-like systems.
+//!
+//! Since what appears to be version 2.18, glibc has shipped the
+//! `__cxa_thread_atexit_impl` symbol which GCC and clang both use to invoke
+//! destructors in C++ thread_local globals. This function does exactly what
+//! we want: it schedules a callback which will be run at thread exit with the
+//! provided argument.
+//!
+//! Unfortunately, our minimum supported glibc version (at the time of writing)
+//! is 2.17, so we can only link this symbol weakly and need to use the
+//! [`list`](super::list) destructor implementation as fallback.
+
+use crate::mem::transmute;
+
+// FIXME: The Rust compiler currently omits weakly function definitions (i.e.,
+// __cxa_thread_atexit_impl) and its metadata from LLVM IR.
+#[no_sanitize(cfi, kcfi)]
+pub unsafe fn register(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
+    /// This is necessary because the __cxa_thread_atexit_impl implementation
+    /// std links to by default may be a C or C++ implementation that was not
+    /// compiled using the Clang integer normalization option.
+    #[cfg(sanitizer_cfi_normalize_integers)]
+    use core::ffi::c_int;
+    #[cfg(not(sanitizer_cfi_normalize_integers))]
+    #[cfi_encoding = "i"]
+    #[repr(transparent)]
+    #[allow(non_camel_case_types)]
+    pub struct c_int(#[allow(dead_code)] pub libc::c_int);
+
+    extern "C" {
+        #[linkage = "extern_weak"]
+        static __dso_handle: *mut u8;
+        #[linkage = "extern_weak"]
+        static __cxa_thread_atexit_impl: Option<
+            extern "C" fn(
+                unsafe extern "C" fn(*mut libc::c_void),
+                *mut libc::c_void,
+                *mut libc::c_void,
+            ) -> c_int,
+        >;
+    }
+
+    if let Some(f) = unsafe { __cxa_thread_atexit_impl } {
+        unsafe {
+            f(
+                transmute::<unsafe extern "C" fn(*mut u8), unsafe extern "C" fn(*mut libc::c_void)>(
+                    dtor,
+                ),
+                t.cast(),
+                core::ptr::addr_of!(__dso_handle) as *mut _,
+            );
+        }
+    } else {
+        unsafe {
+            super::list::register(t, dtor);
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/destructors/list.rs b/library/std/src/sys/thread_local/destructors/list.rs
new file mode 100644
index 00000000000..b9d5214c438
--- /dev/null
+++ b/library/std/src/sys/thread_local/destructors/list.rs
@@ -0,0 +1,44 @@
+use crate::cell::RefCell;
+use crate::sys::thread_local::guard;
+
+#[thread_local]
+static DTORS: RefCell<Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>> = RefCell::new(Vec::new());
+
+pub unsafe fn register(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
+    let Ok(mut dtors) = DTORS.try_borrow_mut() else {
+        // This point can only be reached if the global allocator calls this
+        // function again.
+        // FIXME: maybe use the system allocator instead?
+        rtabort!("the global allocator may not use TLS with destructors");
+    };
+
+    guard::enable();
+
+    dtors.push((t, dtor));
+}
+
+/// The [`guard`] module contains platform-specific functions which will run this
+/// function on thread exit if [`guard::enable`] has been called.
+///
+/// # Safety
+///
+/// May only be run on thread exit to guarantee that there are no live references
+/// to TLS variables while they are destroyed.
+pub unsafe fn run() {
+    loop {
+        let mut dtors = DTORS.borrow_mut();
+        match dtors.pop() {
+            Some((t, dtor)) => {
+                drop(dtors);
+                unsafe {
+                    dtor(t);
+                }
+            }
+            None => {
+                // Free the list memory.
+                *dtors = Vec::new();
+                break;
+            }
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/guard/apple.rs b/library/std/src/sys/thread_local/guard/apple.rs
new file mode 100644
index 00000000000..6c27f7ae35c
--- /dev/null
+++ b/library/std/src/sys/thread_local/guard/apple.rs
@@ -0,0 +1,31 @@
+//! macOS allows registering destructors through _tlv_atexit. But since calling
+//! it while TLS destructors are running is UB, we still need to keep our own
+//! list of destructors.
+
+use crate::cell::Cell;
+use crate::ptr;
+use crate::sys::thread_local::destructors;
+
+pub fn enable() {
+    #[thread_local]
+    static REGISTERED: Cell<bool> = Cell::new(false);
+
+    extern "C" {
+        fn _tlv_atexit(dtor: unsafe extern "C" fn(*mut u8), arg: *mut u8);
+    }
+
+    if !REGISTERED.replace(true) {
+        // SAFETY: Calling _tlv_atexit while TLS destructors are running is UB.
+        // But as run_dtors is only called after being registered, this point
+        // cannot be reached from it.
+        unsafe {
+            _tlv_atexit(run_dtors, ptr::null_mut());
+        }
+    }
+
+    unsafe extern "C" fn run_dtors(_: *mut u8) {
+        unsafe {
+            destructors::run();
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/guard/key.rs b/library/std/src/sys/thread_local/guard/key.rs
new file mode 100644
index 00000000000..e235daf3cc9
--- /dev/null
+++ b/library/std/src/sys/thread_local/guard/key.rs
@@ -0,0 +1,23 @@
+//! A lot of UNIX platforms don't have a way to register TLS destructors.
+//! Instead, we use one TLS key to register a callback which will run
+//! iterate through the destructor list.
+
+use crate::ptr;
+use crate::sys::thread_local::destructors;
+use crate::sys::thread_local::key::StaticKey;
+
+pub fn enable() {
+    static DTORS: StaticKey = StaticKey::new(Some(run));
+
+    // Setting the key value to something other than NULL will result in the
+    // destructor being run at thread exit.
+    unsafe {
+        DTORS.set(ptr::without_provenance_mut(1));
+    }
+
+    unsafe extern "C" fn run(_: *mut u8) {
+        unsafe {
+            destructors::run();
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/guard/solid.rs b/library/std/src/sys/thread_local/guard/solid.rs
new file mode 100644
index 00000000000..b65d00c5b5f
--- /dev/null
+++ b/library/std/src/sys/thread_local/guard/solid.rs
@@ -0,0 +1,23 @@
+//! SOLID, just like macOS, has an API to register TLS destructors. But since
+//! it does not allow specifying an argument to that function, and will not run
+//! destructors for terminated tasks, we still keep our own list.
+
+use crate::cell::Cell;
+use crate::sys::pal::{abi, itron::task};
+use crate::sys::thread_local::destructors;
+
+pub fn enable() {
+    #[thread_local]
+    static REGISTERED: Cell<bool> = Cell::new(false);
+
+    if !REGISTERED.replace(true) {
+        let tid = task::current_task_id_aborting();
+        // Register `tls_dtor` to make sure the TLS destructors are called
+        // for tasks created by other means than `std::thread`
+        unsafe { abi::SOLID_TLS_AddDestructor(tid as i32, tls_dtor) };
+    }
+
+    unsafe extern "C" fn tls_dtor(_unused: *mut u8) {
+        unsafe { destructors::run() };
+    }
+}
diff --git a/library/std/src/sys/thread_local/guard/windows.rs b/library/std/src/sys/thread_local/guard/windows.rs
new file mode 100644
index 00000000000..81797f55170
--- /dev/null
+++ b/library/std/src/sys/thread_local/guard/windows.rs
@@ -0,0 +1,103 @@
+//! Support for Windows TLS destructors.
+//!
+//! Unfortunately, Windows does not provide a nice API to provide a destructor
+//! for a TLS variable. Thus, the solution here ended up being a little more
+//! obscure, but fear not, the internet has informed me [1][2] that this solution
+//! is not unique (no way I could have thought of it as well!). The key idea is
+//! to insert some hook somewhere to run arbitrary code on thread termination.
+//! With this in place we'll be able to run anything we like, including all
+//! TLS destructors!
+//!
+//! In order to realize this, all TLS destructors are tracked by *us*, not the
+//! Windows runtime. This means that we have a global list of destructors for
+//! each TLS key or variable that we know about.
+//!
+//! # What's up with CRT$XLB?
+//!
+//! For anything about TLS destructors to work on Windows, we have to be able
+//! to run *something* when a thread exits. To do so, we place a very special
+//! static in a very special location. If this is encoded in just the right
+//! way, the kernel's loader is apparently nice enough to run some function
+//! of ours whenever a thread exits! How nice of the kernel!
+//!
+//! Lots of detailed information can be found in source [1] above, but the
+//! gist of it is that this is leveraging a feature of Microsoft's PE format
+//! (executable format) which is not actually used by any compilers today.
+//! This apparently translates to any callbacks in the ".CRT$XLB" section
+//! being run on certain events.
+//!
+//! So after all that, we use the compiler's #[link_section] feature to place
+//! a callback pointer into the magic section so it ends up being called.
+//!
+//! # What's up with this callback?
+//!
+//! The callback specified receives a number of parameters from... someone!
+//! (the kernel? the runtime? I'm not quite sure!) There are a few events that
+//! this gets invoked for, but we're currently only interested on when a
+//! thread or a process "detaches" (exits). The process part happens for the
+//! last thread and the thread part happens for any normal thread.
+//!
+//! # The article mentions weird stuff about "/INCLUDE"?
+//!
+//! It sure does! Specifically we're talking about this quote:
+//!
+//! ```quote
+//! The Microsoft run-time library facilitates this process by defining a
+//! memory image of the TLS Directory and giving it the special name
+//! “__tls_used” (Intel x86 platforms) or “_tls_used” (other platforms). The
+//! linker looks for this memory image and uses the data there to create the
+//! TLS Directory. Other compilers that support TLS and work with the
+//! Microsoft linker must use this same technique.
+//! ```
+//!
+//! Basically what this means is that if we want support for our TLS
+//! destructors/our hook being called then we need to make sure the linker does
+//! not omit this symbol. Otherwise it will omit it and our callback won't be
+//! wired up.
+//!
+//! We don't actually use the `/INCLUDE` linker flag here like the article
+//! mentions because the Rust compiler doesn't propagate linker flags, but
+//! instead we use a shim function which performs a volatile 1-byte load from
+//! the address of the symbol to ensure it sticks around.
+//!
+//! [1]: https://www.codeproject.com/Articles/8113/Thread-Local-Storage-The-C-Way
+//! [2]: https://github.com/ChromiumWebApps/chromium/blob/master/base/threading/thread_local_storage_win.cc#L42
+
+use crate::ptr;
+use crate::sys::c;
+
+pub fn enable() {
+    // When destructors are used, we don't want LLVM eliminating CALLBACK for any
+    // reason. Once the symbol makes it to the linker, it will do the rest.
+    unsafe { ptr::from_ref(&CALLBACK).read_volatile() };
+}
+
+#[link_section = ".CRT$XLB"]
+#[cfg_attr(miri, used)] // Miri only considers explicitly `#[used]` statics for `lookup_link_section`
+pub static CALLBACK: unsafe extern "system" fn(c::LPVOID, c::DWORD, c::LPVOID) = tls_callback;
+
+unsafe extern "system" fn tls_callback(_h: c::LPVOID, dw_reason: c::DWORD, _pv: c::LPVOID) {
+    // See comments above for what this is doing. Note that we don't need this
+    // trickery on GNU windows, just on MSVC.
+    #[cfg(all(target_env = "msvc", not(target_thread_local)))]
+    {
+        extern "C" {
+            static _tls_used: u8;
+        }
+
+        unsafe {
+            ptr::from_ref(&_tls_used).read_volatile();
+        }
+    }
+
+    if dw_reason == c::DLL_THREAD_DETACH || dw_reason == c::DLL_PROCESS_DETACH {
+        #[cfg(target_thread_local)]
+        unsafe {
+            super::super::destructors::run();
+        }
+        #[cfg(not(target_thread_local))]
+        unsafe {
+            super::super::key::run_dtors();
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/key/racy.rs b/library/std/src/sys/thread_local/key/racy.rs
new file mode 100644
index 00000000000..e2eaca197d4
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/racy.rs
@@ -0,0 +1,126 @@
+//! An implementation of `const`-creatable TLS keys for non-Windows platforms.
+//!
+//! Most OSs without native TLS will provide a library-based way to create TLS
+//! storage. For each TLS variable, we create a key, which can then be used to
+//! reference an entry in a thread-local table. This then associates each key
+//! with a pointer which we can get and set to store our data.
+//!
+//! Unfortunately, none of these platforms allows creating the key at compile-time,
+//! which means we need a way to lazily create keys (`StaticKey`). Instead of
+//! blocking API like `OnceLock`, we use racy initialization, which should be
+//! more lightweight and avoids circular dependencies with the rest of `std`.
+
+use crate::sync::atomic::{self, AtomicUsize, Ordering};
+
+/// A type for TLS keys that are statically allocated.
+///
+/// This type is entirely `unsafe` to use as it does not protect against
+/// use-after-deallocation or use-during-deallocation.
+///
+/// The actual OS-TLS key is lazily allocated when this is used for the first
+/// time. The key is also deallocated when the Rust runtime exits or `destroy`
+/// is called, whichever comes first.
+///
+/// # Examples
+///
+/// ```ignore (cannot-doctest-private-modules)
+/// use tls::os::{StaticKey, INIT};
+///
+/// // Use a regular global static to store the key.
+/// static KEY: StaticKey = INIT;
+///
+/// // The state provided via `get` and `set` is thread-local.
+/// unsafe {
+///     assert!(KEY.get().is_null());
+///     KEY.set(1 as *mut u8);
+/// }
+/// ```
+pub struct StaticKey {
+    /// Inner static TLS key (internals).
+    key: AtomicUsize,
+    /// Destructor for the TLS value.
+    ///
+    /// See `Key::new` for information about when the destructor runs and how
+    /// it runs.
+    dtor: Option<unsafe extern "C" fn(*mut u8)>,
+}
+
+// Define a sentinel value that is likely not to be returned
+// as a TLS key.
+#[cfg(not(target_os = "nto"))]
+const KEY_SENTVAL: usize = 0;
+// On QNX Neutrino, 0 is always returned when currently not in use.
+// Using 0 would mean to always create two keys and remote the first
+// one (with value of 0) immediately afterwards.
+#[cfg(target_os = "nto")]
+const KEY_SENTVAL: usize = libc::PTHREAD_KEYS_MAX + 1;
+
+impl StaticKey {
+    #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
+    pub const fn new(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> StaticKey {
+        StaticKey { key: atomic::AtomicUsize::new(KEY_SENTVAL), dtor }
+    }
+
+    /// Gets the value associated with this TLS key
+    ///
+    /// This will lazily allocate a TLS key from the OS if one has not already
+    /// been allocated.
+    #[inline]
+    pub unsafe fn get(&self) -> *mut u8 {
+        unsafe { super::get(self.key()) }
+    }
+
+    /// Sets this TLS key to a new value.
+    ///
+    /// This will lazily allocate a TLS key from the OS if one has not already
+    /// been allocated.
+    #[inline]
+    pub unsafe fn set(&self, val: *mut u8) {
+        unsafe { super::set(self.key(), val) }
+    }
+
+    #[inline]
+    fn key(&self) -> super::Key {
+        match self.key.load(Ordering::Acquire) {
+            KEY_SENTVAL => self.lazy_init() as super::Key,
+            n => n as super::Key,
+        }
+    }
+
+    fn lazy_init(&self) -> usize {
+        // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange
+        // below relies on using KEY_SENTVAL as a sentinel value to check who won the
+        // race to set the shared TLS key. As far as I know, there is no
+        // guaranteed value that cannot be returned as a posix_key_create key,
+        // so there is no value we can initialize the inner key with to
+        // prove that it has not yet been set. As such, we'll continue using a
+        // value of KEY_SENTVAL, but with some gyrations to make sure we have a non-KEY_SENTVAL
+        // value returned from the creation routine.
+        // FIXME: this is clearly a hack, and should be cleaned up.
+        let key1 = super::create(self.dtor);
+        let key = if key1 as usize != KEY_SENTVAL {
+            key1
+        } else {
+            let key2 = super::create(self.dtor);
+            unsafe {
+                super::destroy(key1);
+            }
+            key2
+        };
+        rtassert!(key as usize != KEY_SENTVAL);
+        match self.key.compare_exchange(
+            KEY_SENTVAL,
+            key as usize,
+            Ordering::Release,
+            Ordering::Acquire,
+        ) {
+            // The CAS succeeded, so we've created the actual key
+            Ok(_) => key as usize,
+            // If someone beat us to the punch, use their key instead
+            Err(n) => unsafe {
+                super::destroy(key);
+                n
+            },
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/key/sgx.rs b/library/std/src/sys/thread_local/key/sgx.rs
new file mode 100644
index 00000000000..4aa2e5afa72
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/sgx.rs
@@ -0,0 +1,23 @@
+use crate::sys::pal::abi::tls::{Key as AbiKey, Tls};
+
+pub type Key = usize;
+
+#[inline]
+pub fn create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key {
+    Tls::create(dtor).as_usize()
+}
+
+#[inline]
+pub unsafe fn set(key: Key, value: *mut u8) {
+    Tls::set(AbiKey::from_usize(key), value)
+}
+
+#[inline]
+pub unsafe fn get(key: Key) -> *mut u8 {
+    Tls::get(AbiKey::from_usize(key))
+}
+
+#[inline]
+pub unsafe fn destroy(key: Key) {
+    Tls::destroy(AbiKey::from_usize(key))
+}
diff --git a/library/std/src/sys/thread_local/key/tests.rs b/library/std/src/sys/thread_local/key/tests.rs
new file mode 100644
index 00000000000..24cad396da2
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/tests.rs
@@ -0,0 +1,53 @@
+use super::StaticKey;
+use crate::ptr;
+
+#[test]
+fn smoke() {
+    static K1: StaticKey = StaticKey::new(None);
+    static K2: StaticKey = StaticKey::new(None);
+
+    unsafe {
+        assert!(K1.get().is_null());
+        assert!(K2.get().is_null());
+        K1.set(ptr::without_provenance_mut(1));
+        K2.set(ptr::without_provenance_mut(2));
+        assert_eq!(K1.get() as usize, 1);
+        assert_eq!(K2.get() as usize, 2);
+    }
+}
+
+#[test]
+fn destructors() {
+    use crate::mem::ManuallyDrop;
+    use crate::sync::Arc;
+    use crate::thread;
+
+    unsafe extern "C" fn destruct(ptr: *mut u8) {
+        drop(unsafe { Arc::from_raw(ptr as *const ()) });
+    }
+
+    static KEY: StaticKey = StaticKey::new(Some(destruct));
+
+    let shared1 = Arc::new(());
+    let shared2 = Arc::clone(&shared1);
+
+    unsafe {
+        assert!(KEY.get().is_null());
+        KEY.set(Arc::into_raw(shared1) as *mut u8);
+    }
+
+    thread::spawn(move || unsafe {
+        assert!(KEY.get().is_null());
+        KEY.set(Arc::into_raw(shared2) as *mut u8);
+    })
+    .join()
+    .unwrap();
+
+    // Leak the Arc, let the TLS destructor clean it up.
+    let shared1 = unsafe { ManuallyDrop::new(Arc::from_raw(KEY.get() as *const ())) };
+    assert_eq!(
+        Arc::strong_count(&shared1),
+        1,
+        "destructor should have dropped the other reference on thread exit"
+    );
+}
diff --git a/library/std/src/sys/thread_local/key/unix.rs b/library/std/src/sys/thread_local/key/unix.rs
new file mode 100644
index 00000000000..13522d44b35
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/unix.rs
@@ -0,0 +1,27 @@
+use crate::mem;
+
+pub type Key = libc::pthread_key_t;
+
+#[inline]
+pub fn create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key {
+    let mut key = 0;
+    assert_eq!(unsafe { libc::pthread_key_create(&mut key, mem::transmute(dtor)) }, 0);
+    key
+}
+
+#[inline]
+pub unsafe fn set(key: Key, value: *mut u8) {
+    let r = unsafe { libc::pthread_setspecific(key, value as *mut _) };
+    debug_assert_eq!(r, 0);
+}
+
+#[inline]
+pub unsafe fn get(key: Key) -> *mut u8 {
+    unsafe { libc::pthread_getspecific(key) as *mut u8 }
+}
+
+#[inline]
+pub unsafe fn destroy(key: Key) {
+    let r = unsafe { libc::pthread_key_delete(key) };
+    debug_assert_eq!(r, 0);
+}
diff --git a/library/std/src/sys/thread_local/key/windows.rs b/library/std/src/sys/thread_local/key/windows.rs
new file mode 100644
index 00000000000..ad0e72c29ed
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/windows.rs
@@ -0,0 +1,206 @@
+//! Implementation of `StaticKey` for Windows.
+//!
+//! Windows has no native support for running destructors so we manage our own
+//! list of destructors to keep track of how to destroy keys. We then install a
+//! callback later to get invoked whenever a thread exits, running all
+//! appropriate destructors (see the [`guard`](guard) module documentation).
+//!
+//! This will likely need to be improved over time, but this module attempts a
+//! "poor man's" destructor callback system. Once we've got a list of what to
+//! run, we iterate over all keys, check their values, and then run destructors
+//! if the values turn out to be non null (setting them to null just beforehand).
+//! We do this a few times in a loop to basically match Unix semantics. If we
+//! don't reach a fixed point after a short while then we just inevitably leak
+//! something.
+//!
+//! The list is implemented as an atomic single-linked list of `StaticKey`s and
+//! does not support unregistration. Unfortunately, this means that we cannot
+//! use racy initialization for creating the keys in `StaticKey`, as that could
+//! result in destructors being missed. Hence, we synchronize the creation of
+//! keys with destructors through [`INIT_ONCE`](c::INIT_ONCE) (`std`'s
+//! [`Once`](crate::sync::Once) cannot be used since it might use TLS itself).
+//! For keys without destructors, racy initialization suffices.
+
+// FIXME: investigate using a fixed-size array instead, as the maximum number
+//        of keys is [limited to 1088](https://learn.microsoft.com/en-us/windows/win32/ProcThread/thread-local-storage).
+
+use crate::cell::UnsafeCell;
+use crate::ptr;
+use crate::sync::atomic::{
+    AtomicPtr, AtomicU32,
+    Ordering::{AcqRel, Acquire, Relaxed, Release},
+};
+use crate::sys::c;
+use crate::sys::thread_local::guard;
+
+type Key = c::DWORD;
+type Dtor = unsafe extern "C" fn(*mut u8);
+
+pub struct StaticKey {
+    /// The key value shifted up by one. Since TLS_OUT_OF_INDEXES == DWORD::MAX
+    /// is not a valid key value, this allows us to use zero as sentinel value
+    /// without risking overflow.
+    key: AtomicU32,
+    dtor: Option<Dtor>,
+    next: AtomicPtr<StaticKey>,
+    /// Currently, destructors cannot be unregistered, so we cannot use racy
+    /// initialization for keys. Instead, we need synchronize initialization.
+    /// Use the Windows-provided `Once` since it does not require TLS.
+    once: UnsafeCell<c::INIT_ONCE>,
+}
+
+impl StaticKey {
+    #[inline]
+    pub const fn new(dtor: Option<Dtor>) -> StaticKey {
+        StaticKey {
+            key: AtomicU32::new(0),
+            dtor,
+            next: AtomicPtr::new(ptr::null_mut()),
+            once: UnsafeCell::new(c::INIT_ONCE_STATIC_INIT),
+        }
+    }
+
+    #[inline]
+    pub unsafe fn set(&'static self, val: *mut u8) {
+        let r = unsafe { c::TlsSetValue(self.key(), val.cast()) };
+        debug_assert_eq!(r, c::TRUE);
+    }
+
+    #[inline]
+    pub unsafe fn get(&'static self) -> *mut u8 {
+        unsafe { c::TlsGetValue(self.key()).cast() }
+    }
+
+    #[inline]
+    fn key(&'static self) -> Key {
+        match self.key.load(Acquire) {
+            0 => unsafe { self.init() },
+            key => key - 1,
+        }
+    }
+
+    #[cold]
+    unsafe fn init(&'static self) -> Key {
+        if self.dtor.is_some() {
+            let mut pending = c::FALSE;
+            let r = unsafe {
+                c::InitOnceBeginInitialize(self.once.get(), 0, &mut pending, ptr::null_mut())
+            };
+            assert_eq!(r, c::TRUE);
+
+            if pending == c::FALSE {
+                // Some other thread initialized the key, load it.
+                self.key.load(Relaxed) - 1
+            } else {
+                let key = unsafe { c::TlsAlloc() };
+                if key == c::TLS_OUT_OF_INDEXES {
+                    // Wakeup the waiting threads before panicking to avoid deadlock.
+                    unsafe {
+                        c::InitOnceComplete(
+                            self.once.get(),
+                            c::INIT_ONCE_INIT_FAILED,
+                            ptr::null_mut(),
+                        );
+                    }
+                    panic!("out of TLS indexes");
+                }
+
+                unsafe {
+                    register_dtor(self);
+                }
+
+                // Release-storing the key needs to be the last thing we do.
+                // This is because in `fn key()`, other threads will do an acquire load of the key,
+                // and if that sees this write then it will entirely bypass the `InitOnce`. We thus
+                // need to establish synchronization through `key`. In particular that acquire load
+                // must happen-after the register_dtor above, to ensure the dtor actually runs!
+                self.key.store(key + 1, Release);
+
+                let r = unsafe { c::InitOnceComplete(self.once.get(), 0, ptr::null_mut()) };
+                debug_assert_eq!(r, c::TRUE);
+
+                key
+            }
+        } else {
+            // If there is no destructor to clean up, we can use racy initialization.
+
+            let key = unsafe { c::TlsAlloc() };
+            assert_ne!(key, c::TLS_OUT_OF_INDEXES, "out of TLS indexes");
+
+            match self.key.compare_exchange(0, key + 1, AcqRel, Acquire) {
+                Ok(_) => key,
+                Err(new) => unsafe {
+                    // Some other thread completed initialization first, so destroy
+                    // our key and use theirs.
+                    let r = c::TlsFree(key);
+                    debug_assert_eq!(r, c::TRUE);
+                    new - 1
+                },
+            }
+        }
+    }
+}
+
+unsafe impl Send for StaticKey {}
+unsafe impl Sync for StaticKey {}
+
+static DTORS: AtomicPtr<StaticKey> = AtomicPtr::new(ptr::null_mut());
+
+/// Should only be called once per key, otherwise loops or breaks may occur in
+/// the linked list.
+unsafe fn register_dtor(key: &'static StaticKey) {
+    guard::enable();
+
+    let this = <*const StaticKey>::cast_mut(key);
+    // Use acquire ordering to pass along the changes done by the previously
+    // registered keys when we store the new head with release ordering.
+    let mut head = DTORS.load(Acquire);
+    loop {
+        key.next.store(head, Relaxed);
+        match DTORS.compare_exchange_weak(head, this, Release, Acquire) {
+            Ok(_) => break,
+            Err(new) => head = new,
+        }
+    }
+}
+
+/// This will and must only be run by the destructor callback in [`guard`].
+pub unsafe fn run_dtors() {
+    for _ in 0..5 {
+        let mut any_run = false;
+
+        // Use acquire ordering to observe key initialization.
+        let mut cur = DTORS.load(Acquire);
+        while !cur.is_null() {
+            let pre_key = unsafe { (*cur).key.load(Acquire) };
+            let dtor = unsafe { (*cur).dtor.unwrap() };
+            cur = unsafe { (*cur).next.load(Relaxed) };
+
+            // In StaticKey::init, we register the dtor before setting `key`.
+            // So if one thread's `run_dtors` races with another thread executing `init` on the same
+            // `StaticKey`, we can encounter a key of 0 here. That means this key was never
+            // initialized in this thread so we can safely skip it.
+            if pre_key == 0 {
+                continue;
+            }
+            // If this is non-zero, then via the `Acquire` load above we synchronized with
+            // everything relevant for this key. (It's not clear that this is needed, since the
+            // release-acquire pair on DTORS also establishes synchronization, but better safe than
+            // sorry.)
+            let key = pre_key - 1;
+
+            let ptr = unsafe { c::TlsGetValue(key) };
+            if !ptr.is_null() {
+                unsafe {
+                    c::TlsSetValue(key, ptr::null_mut());
+                    dtor(ptr as *mut _);
+                    any_run = true;
+                }
+            }
+        }
+
+        if !any_run {
+            break;
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/key/xous.rs b/library/std/src/sys/thread_local/key/xous.rs
new file mode 100644
index 00000000000..a23f6de95f7
--- /dev/null
+++ b/library/std/src/sys/thread_local/key/xous.rs
@@ -0,0 +1,216 @@
+//! Thread Local Storage
+//!
+//! Currently, we are limited to 1023 TLS entries. The entries
+//! live in a page of memory that's unique per-process, and is
+//! stored in the `$tp` register. If this register is 0, then
+//! TLS has not been initialized and thread cleanup can be skipped.
+//!
+//! The index into this register is the `key`. This key is identical
+//! between all threads, but indexes a different offset within this
+//! pointer.
+//!
+//! # Dtor registration (stolen from Windows)
+//!
+//! Xous has no native support for running destructors so we manage our own
+//! list of destructors to keep track of how to destroy keys. When a thread
+//! or the process exits, `run_dtors` is called, which will iterate through
+//! the list and run the destructors.
+//!
+//! Currently unregistration from this list is not supported. A destructor can be
+//! registered but cannot be unregistered. There's various simplifying reasons
+//! for doing this, the big ones being:
+//!
+//! 1. Currently we don't even support deallocating TLS keys, so normal operation
+//!    doesn't need to deallocate a destructor.
+//! 2. There is no point in time where we know we can unregister a destructor
+//!    because it could always be getting run by some remote thread.
+//!
+//! Typically processes have a statically known set of TLS keys which is pretty
+//! small, and we'd want to keep this memory alive for the whole process anyway
+//! really.
+//!
+//! Perhaps one day we can fold the `Box` here into a static allocation,
+//! expanding the `StaticKey` structure to contain not only a slot for the TLS
+//! key but also a slot for the destructor queue on windows. An optimization for
+//! another day!
+
+// FIXME(joboet): implement support for native TLS instead.
+
+use crate::mem::ManuallyDrop;
+use crate::ptr;
+use crate::sync::atomic::AtomicPtr;
+use crate::sync::atomic::AtomicUsize;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use core::arch::asm;
+
+use crate::os::xous::ffi::{map_memory, unmap_memory, MemoryFlags};
+
+pub type Key = usize;
+pub type Dtor = unsafe extern "C" fn(*mut u8);
+
+const TLS_MEMORY_SIZE: usize = 4096;
+
+/// TLS keys start at `1`. Index `0` is unused
+#[cfg(not(test))]
+#[export_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key13TLS_KEY_INDEXE"]
+static TLS_KEY_INDEX: AtomicUsize = AtomicUsize::new(1);
+
+#[cfg(not(test))]
+#[export_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key9DTORSE"]
+static DTORS: AtomicPtr<Node> = AtomicPtr::new(ptr::null_mut());
+
+#[cfg(test)]
+extern "Rust" {
+    #[link_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key13TLS_KEY_INDEXE"]
+    static TLS_KEY_INDEX: AtomicUsize;
+
+    #[link_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key9DTORSE"]
+    static DTORS: AtomicPtr<Node>;
+}
+
+fn tls_ptr_addr() -> *mut *mut u8 {
+    let mut tp: usize;
+    unsafe {
+        asm!(
+            "mv {}, tp",
+            out(reg) tp,
+        );
+    }
+    core::ptr::with_exposed_provenance_mut::<*mut u8>(tp)
+}
+
+/// Create an area of memory that's unique per thread. This area will
+/// contain all thread local pointers.
+fn tls_table() -> &'static mut [*mut u8] {
+    let tp = tls_ptr_addr();
+
+    if !tp.is_null() {
+        return unsafe {
+            core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / core::mem::size_of::<*mut u8>())
+        };
+    }
+    // If the TP register is `0`, then this thread hasn't initialized
+    // its TLS yet. Allocate a new page to store this memory.
+    let tp = unsafe {
+        map_memory(
+            None,
+            None,
+            TLS_MEMORY_SIZE / core::mem::size_of::<*mut u8>(),
+            MemoryFlags::R | MemoryFlags::W,
+        )
+        .expect("Unable to allocate memory for thread local storage")
+    };
+
+    for val in tp.iter() {
+        assert!(*val as usize == 0);
+    }
+
+    unsafe {
+        // Set the thread's `$tp` register
+        asm!(
+            "mv tp, {}",
+            in(reg) tp.as_mut_ptr() as usize,
+        );
+    }
+    tp
+}
+
+#[inline]
+pub fn create(dtor: Option<Dtor>) -> Key {
+    // Allocate a new TLS key. These keys are shared among all threads.
+    #[allow(unused_unsafe)]
+    let key = unsafe { TLS_KEY_INDEX.fetch_add(1, Relaxed) };
+    if let Some(f) = dtor {
+        unsafe { register_dtor(key, f) };
+    }
+    key
+}
+
+#[inline]
+pub unsafe fn set(key: Key, value: *mut u8) {
+    assert!((key < 1022) && (key >= 1));
+    let table = tls_table();
+    table[key] = value;
+}
+
+#[inline]
+pub unsafe fn get(key: Key) -> *mut u8 {
+    assert!((key < 1022) && (key >= 1));
+    tls_table()[key]
+}
+
+#[inline]
+pub unsafe fn destroy(_key: Key) {
+    // Just leak the key. Probably not great on long-running systems that create
+    // lots of TLS variables, but in practice that's not an issue.
+}
+
+struct Node {
+    dtor: Dtor,
+    key: Key,
+    next: *mut Node,
+}
+
+unsafe fn register_dtor(key: Key, dtor: Dtor) {
+    let mut node = ManuallyDrop::new(Box::new(Node { key, dtor, next: ptr::null_mut() }));
+
+    #[allow(unused_unsafe)]
+    let mut head = unsafe { DTORS.load(Acquire) };
+    loop {
+        node.next = head;
+        #[allow(unused_unsafe)]
+        match unsafe { DTORS.compare_exchange(head, &mut **node, Release, Acquire) } {
+            Ok(_) => return, // nothing to drop, we successfully added the node to the list
+            Err(cur) => head = cur,
+        }
+    }
+}
+
+pub unsafe fn destroy_tls() {
+    let tp = tls_ptr_addr();
+
+    // If the pointer address is 0, then this thread has no TLS.
+    if tp.is_null() {
+        return;
+    }
+
+    unsafe { run_dtors() };
+
+    // Finally, free the TLS array
+    unsafe {
+        unmap_memory(core::slice::from_raw_parts_mut(
+            tp,
+            TLS_MEMORY_SIZE / core::mem::size_of::<usize>(),
+        ))
+        .unwrap()
+    };
+}
+
+unsafe fn run_dtors() {
+    let mut any_run = true;
+
+    // Run the destructor "some" number of times. This is 5x on Windows,
+    // so we copy it here. This allows TLS variables to create new
+    // TLS variables upon destruction that will also get destroyed.
+    // Keep going until we run out of tries or until we have nothing
+    // left to destroy.
+    for _ in 0..5 {
+        if !any_run {
+            break;
+        }
+        any_run = false;
+        #[allow(unused_unsafe)]
+        let mut cur = unsafe { DTORS.load(Acquire) };
+        while !cur.is_null() {
+            let ptr = unsafe { get((*cur).key) };
+
+            if !ptr.is_null() {
+                unsafe { set((*cur).key, ptr::null_mut()) };
+                unsafe { ((*cur).dtor)(ptr as *mut _) };
+                any_run = true;
+            }
+
+            unsafe { cur = (*cur).next };
+        }
+    }
+}
diff --git a/library/std/src/sys/thread_local/mod.rs b/library/std/src/sys/thread_local/mod.rs
index 0a78a1a1cf0..6e3d8382b42 100644
--- a/library/std/src/sys/thread_local/mod.rs
+++ b/library/std/src/sys/thread_local/mod.rs
@@ -1,27 +1,135 @@
-#![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")]
-#![cfg_attr(test, allow(unused))]
+//! Implementation of the `thread_local` macro.
+//!
+//! There are three different thread-local implementations:
+//! * Some targets lack threading support, and hence have only one thread, so
+//!   the TLS data is stored in a normal `static`.
+//! * Some targets support TLS natively via the dynamic linker and C runtime.
+//! * On some targets, the OS provides a library-based TLS implementation. The
+//!   TLS data is heap-allocated and referenced using a TLS key.
+//!
+//! Each implementation provides a macro which generates the `LocalKey` `const`
+//! used to reference the TLS variable, along with the necessary helper structs
+//! to track the initialization/destruction state of the variable.
+//!
+//! Additionally, this module contains abstractions for the OS interfaces used
+//! for these implementations.
 
-// There are three thread-local implementations: "static", "fast", "OS".
-// The "OS" thread local key type is accessed via platform-specific API calls and is slow, while the
-// "fast" key type is accessed via code generated via LLVM, where TLS keys are set up by the linker.
-// "static" is for single-threaded platforms where a global static is sufficient.
+#![cfg_attr(test, allow(unused))]
+#![doc(hidden)]
+#![forbid(unsafe_op_in_unsafe_fn)]
+#![unstable(
+    feature = "thread_local_internals",
+    reason = "internal details of the thread_local macro",
+    issue = "none"
+)]
 
 cfg_if::cfg_if! {
-    if #[cfg(any(all(target_family = "wasm", not(target_feature = "atomics")), target_os = "uefi"))] {
-        #[doc(hidden)]
-        mod static_local;
-        #[doc(hidden)]
-        pub use static_local::{EagerStorage, LazyStorage, thread_local_inner};
+    if #[cfg(any(
+        all(target_family = "wasm", not(target_feature = "atomics")),
+        target_os = "uefi",
+        target_os = "zkvm",
+    ))] {
+        mod statik;
+        pub use statik::{EagerStorage, LazyStorage, thread_local_inner};
     } else if #[cfg(target_thread_local)] {
-        #[doc(hidden)]
-        mod fast_local;
-        #[doc(hidden)]
-        pub use fast_local::{EagerStorage, LazyStorage, thread_local_inner};
+        mod native;
+        pub use native::{EagerStorage, LazyStorage, thread_local_inner};
     } else {
-        #[doc(hidden)]
-        mod os_local;
-        #[doc(hidden)]
-        pub use os_local::{Key, thread_local_inner};
+        mod os;
+        pub use os::{Key, thread_local_inner};
+    }
+}
+
+/// This module maintains a list of TLS destructors for the current thread,
+/// all of which will be run on thread exit.
+pub(crate) mod destructors {
+    cfg_if::cfg_if! {
+        if #[cfg(all(
+            target_thread_local,
+            any(
+                target_os = "linux",
+                target_os = "android",
+                target_os = "fuchsia",
+                target_os = "redox",
+                target_os = "hurd",
+                target_os = "netbsd",
+                target_os = "dragonfly"
+            )
+        ))] {
+            mod linux;
+            mod list;
+            pub(super) use linux::register;
+            pub(super) use list::run;
+        } else if #[cfg(all(
+            target_thread_local,
+            not(all(target_family = "wasm", not(target_feature = "atomics")))
+        ))] {
+            mod list;
+            pub(super) use list::register;
+            pub(crate) use list::run;
+        }
+    }
+}
+
+/// This module provides a way to schedule the execution of the destructor list
+/// on systems without a per-variable destructor system.
+mod guard {
+    cfg_if::cfg_if! {
+        if #[cfg(all(target_thread_local, target_vendor = "apple"))] {
+            mod apple;
+            pub(super) use apple::enable;
+        } else if #[cfg(target_os = "windows")] {
+            mod windows;
+            pub(super) use windows::enable;
+        } else if #[cfg(any(
+            all(target_family = "wasm", target_feature = "atomics"),
+            target_os = "hermit",
+        ))] {
+            pub(super) fn enable() {}
+        } else if #[cfg(target_os = "solid_asp3")] {
+            mod solid;
+            pub(super) use solid::enable;
+        } else if #[cfg(all(target_thread_local, not(target_family = "wasm")))] {
+            mod key;
+            pub(super) use key::enable;
+        }
+    }
+}
+
+/// This module provides the `StaticKey` abstraction over OS TLS keys.
+pub(crate) mod key {
+    cfg_if::cfg_if! {
+        if #[cfg(any(
+            all(not(target_vendor = "apple"), target_family = "unix"),
+            target_os = "teeos",
+        ))] {
+            mod racy;
+            mod unix;
+            #[cfg(test)]
+            mod tests;
+            pub(super) use racy::StaticKey;
+            use unix::{Key, create, destroy, get, set};
+        } else if #[cfg(all(not(target_thread_local), target_os = "windows"))] {
+            #[cfg(test)]
+            mod tests;
+            mod windows;
+            pub(super) use windows::{StaticKey, run_dtors};
+        } else if #[cfg(all(target_vendor = "fortanix", target_env = "sgx"))] {
+            mod racy;
+            mod sgx;
+            #[cfg(test)]
+            mod tests;
+            pub(super) use racy::StaticKey;
+            use sgx::{Key, create, destroy, get, set};
+        } else if #[cfg(target_os = "xous")] {
+            mod racy;
+            #[cfg(test)]
+            mod tests;
+            mod xous;
+            pub(super) use racy::StaticKey;
+            pub(crate) use xous::destroy_tls;
+            use xous::{Key, create, destroy, get, set};
+        }
     }
 }
 
diff --git a/library/std/src/sys/thread_local/fast_local/eager.rs b/library/std/src/sys/thread_local/native/eager.rs
index b97bd9cc88c..99e5ae7fb96 100644
--- a/library/std/src/sys/thread_local/fast_local/eager.rs
+++ b/library/std/src/sys/thread_local/native/eager.rs
@@ -1,7 +1,7 @@
 use crate::cell::{Cell, UnsafeCell};
 use crate::ptr::{self, drop_in_place};
 use crate::sys::thread_local::abort_on_dtor_unwind;
-use crate::sys::thread_local_dtor::register_dtor;
+use crate::sys::thread_local::destructors;
 
 #[derive(Clone, Copy)]
 enum State {
@@ -45,7 +45,7 @@ impl<T> Storage<T> {
         // SAFETY:
         // The caller guarantees that `self` will be valid until thread destruction.
         unsafe {
-            register_dtor(ptr::from_ref(self).cast_mut().cast(), destroy::<T>);
+            destructors::register(ptr::from_ref(self).cast_mut().cast(), destroy::<T>);
         }
 
         self.state.set(State::Alive);
diff --git a/library/std/src/sys/thread_local/fast_local/lazy.rs b/library/std/src/sys/thread_local/native/lazy.rs
index c1ada35d484..9d47e8ef689 100644
--- a/library/std/src/sys/thread_local/fast_local/lazy.rs
+++ b/library/std/src/sys/thread_local/native/lazy.rs
@@ -2,7 +2,7 @@ use crate::cell::UnsafeCell;
 use crate::hint::unreachable_unchecked;
 use crate::ptr;
 use crate::sys::thread_local::abort_on_dtor_unwind;
-use crate::sys::thread_local_dtor::register_dtor;
+use crate::sys::thread_local::destructors;
 
 pub unsafe trait DestroyedState: Sized {
     fn register_dtor<T>(s: &Storage<T, Self>);
@@ -15,7 +15,7 @@ unsafe impl DestroyedState for ! {
 unsafe impl DestroyedState for () {
     fn register_dtor<T>(s: &Storage<T, ()>) {
         unsafe {
-            register_dtor(ptr::from_ref(s).cast_mut().cast(), destroy::<T>);
+            destructors::register(ptr::from_ref(s).cast_mut().cast(), destroy::<T>);
         }
     }
 }
diff --git a/library/std/src/sys/thread_local/fast_local/mod.rs b/library/std/src/sys/thread_local/native/mod.rs
index 575d60de4ee..1cc45fe892d 100644
--- a/library/std/src/sys/thread_local/fast_local/mod.rs
+++ b/library/std/src/sys/thread_local/native/mod.rs
@@ -29,8 +29,6 @@
 //! eliminates the `Destroyed` state for these values, which can allow more niche
 //! optimizations to occur for the `State` enum. For `Drop` types, `()` is used.
 
-#![deny(unsafe_op_in_unsafe_fn)]
-
 mod eager;
 mod lazy;
 
diff --git a/library/std/src/sys/thread_local/os_local.rs b/library/std/src/sys/thread_local/os.rs
index ee5adef66ea..6980c897fdb 100644
--- a/library/std/src/sys/thread_local/os_local.rs
+++ b/library/std/src/sys/thread_local/os.rs
@@ -2,7 +2,7 @@ use super::abort_on_dtor_unwind;
 use crate::cell::Cell;
 use crate::marker::PhantomData;
 use crate::ptr;
-use crate::sys_common::thread_local_key::StaticKey as OsKey;
+use crate::sys::thread_local::key::StaticKey as OsKey;
 
 #[doc(hidden)]
 #[allow_internal_unstable(thread_local_internals)]
diff --git a/library/std/src/sys/thread_local/static_local.rs b/library/std/src/sys/thread_local/statik.rs
index 0f08cab1ae4..0f08cab1ae4 100644
--- a/library/std/src/sys/thread_local/static_local.rs
+++ b/library/std/src/sys/thread_local/statik.rs