diff options
| author | bors <bors@rust-lang.org> | 2024-06-24 15:55:28 +0000 |
|---|---|---|
| committer | bors <bors@rust-lang.org> | 2024-06-24 15:55:28 +0000 |
| commit | 5a3e2a4e921097c8f2bf6ea7565f8abe878cdbd4 (patch) | |
| tree | 8fef4eec253cbf00799f8a29b3f3a5e01fe4866b /library/std/src/sys/pal | |
| parent | d371d17496f2ce3a56da76aa083f4ef157572c20 (diff) | |
| parent | 50a02ed789ed164f09d3ec8471f71c9c2b02df1d (diff) | |
Auto merge of #126523 - joboet:the_great_big_tls_refactor, r=Mark-Simulacrum
std: refactor the TLS implementation As discovered by Mara in #110897, our TLS implementation is a total mess. In the past months, I have simplified the actual macros and their expansions, but the majority of the complexity comes from the platform-specific support code needed to create keys and register destructors. In keeping with #117276, I have therefore moved all of the `thread_local_key`/`thread_local_dtor` modules to the `thread_local` module in `sys` and merged them into a new structure, so that future porters of `std` can simply mix-and-match the existing code instead of having to copy the same (bad) implementation everywhere. The new structure should become obvious when looking at `sys/thread_local/mod.rs`. Unfortunately, the documentation changes associated with the refactoring have made this PR rather large. That said, this contains no functional changes except for two small ones: * the key-based destructor fallback now, by virtue of sharing the implementation used by macOS and others, stores its list in a `#[thread_local]` static instead of in the key, eliminating one indirection layer and drastically simplifying its code. * I've switched over ZKVM (tier 3) to use the same implementation as WebAssembly, as the implementation was just a way worse version of that Please let me know if I can make this easier to review! I know these large PRs aren't optimal, but I couldn't think of any good intermediate steps. `@rustbot` label +A-thread-locals
Diffstat (limited to 'library/std/src/sys/pal')
31 files changed, 5 insertions, 998 deletions
diff --git a/library/std/src/sys/pal/hermit/mod.rs b/library/std/src/sys/pal/hermit/mod.rs index eca7351d54c..55583b89d67 100644 --- a/library/std/src/sys/pal/hermit/mod.rs +++ b/library/std/src/sys/pal/hermit/mod.rs @@ -32,9 +32,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -pub mod thread_local_dtor; -#[path = "../unsupported/thread_local_key.rs"] -pub mod thread_local_key; pub mod time; use crate::io::ErrorKind; @@ -97,7 +94,6 @@ pub unsafe extern "C" fn runtime_entry( argv: *const *const c_char, env: *const *const c_char, ) -> ! { - use thread_local_dtor::run_dtors; extern "C" { fn main(argc: isize, argv: *const *const c_char) -> i32; } @@ -107,7 +103,7 @@ pub unsafe extern "C" fn runtime_entry( let result = main(argc as isize, argv); - run_dtors(); + crate::sys::thread_local::destructors::run(); hermit_abi::exit(result); } diff --git a/library/std/src/sys/pal/hermit/thread.rs b/library/std/src/sys/pal/hermit/thread.rs index 07a843a597e..a244b953d2a 100644 --- a/library/std/src/sys/pal/hermit/thread.rs +++ b/library/std/src/sys/pal/hermit/thread.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] use super::hermit_abi; -use super::thread_local_dtor::run_dtors; use crate::ffi::CStr; use crate::io; use crate::mem; @@ -50,7 +49,7 @@ impl Thread { Box::from_raw(ptr::with_exposed_provenance::<Box<dyn FnOnce()>>(main).cast_mut())(); // run all destructors - run_dtors(); + crate::sys::thread_local::destructors::run(); } } } diff --git a/library/std/src/sys/pal/hermit/thread_local_dtor.rs b/library/std/src/sys/pal/hermit/thread_local_dtor.rs deleted file mode 100644 index 98adaf4bff1..00000000000 --- a/library/std/src/sys/pal/hermit/thread_local_dtor.rs +++ /dev/null @@ -1,29 +0,0 @@ -#![cfg(target_thread_local)] -#![unstable(feature = "thread_local_internals", issue = "none")] - -// Simplify dtor registration by using a list of destructors. -// The this solution works like the implementation of macOS and -// doesn't additional OS support - -use crate::cell::RefCell; - -#[thread_local] -static DTORS: RefCell<Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>> = RefCell::new(Vec::new()); - -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - match DTORS.try_borrow_mut() { - Ok(mut dtors) => dtors.push((t, dtor)), - Err(_) => rtabort!("global allocator may not use TLS"), - } -} - -// every thread call this function to run through all possible destructors -pub unsafe fn run_dtors() { - let mut list = DTORS.take(); - while !list.is_empty() { - for (ptr, dtor) in list { - dtor(ptr); - } - list = DTORS.take(); - } -} diff --git a/library/std/src/sys/pal/itron/thread.rs b/library/std/src/sys/pal/itron/thread.rs index a0c8217128f..fd7b5558f75 100644 --- a/library/std/src/sys/pal/itron/thread.rs +++ b/library/std/src/sys/pal/itron/thread.rs @@ -15,7 +15,6 @@ use crate::{ num::NonZero, ptr::NonNull, sync::atomic::{AtomicUsize, Ordering}, - sys::thread_local_dtor::run_dtors, time::Duration, }; @@ -117,7 +116,7 @@ impl Thread { // Run TLS destructors now because they are not // called automatically for terminated tasks. - unsafe { run_dtors() }; + unsafe { crate::sys::thread_local::destructors::run() }; let old_lifecycle = inner .lifecycle diff --git a/library/std/src/sys/pal/sgx/mod.rs b/library/std/src/sys/pal/sgx/mod.rs index d30976ec151..851ab9b9f97 100644 --- a/library/std/src/sys/pal/sgx/mod.rs +++ b/library/std/src/sys/pal/sgx/mod.rs @@ -26,7 +26,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -pub mod thread_local_key; pub mod thread_parking; pub mod time; pub mod waitqueue; diff --git a/library/std/src/sys/pal/sgx/thread_local_key.rs b/library/std/src/sys/pal/sgx/thread_local_key.rs deleted file mode 100644 index c7a57d3a3d4..00000000000 --- a/library/std/src/sys/pal/sgx/thread_local_key.rs +++ /dev/null @@ -1,23 +0,0 @@ -use super::abi::tls::{Key as AbiKey, Tls}; - -pub type Key = usize; - -#[inline] -pub unsafe fn create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key { - Tls::create(dtor).as_usize() -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - Tls::set(AbiKey::from_usize(key), value) -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - Tls::get(AbiKey::from_usize(key)) -} - -#[inline] -pub unsafe fn destroy(key: Key) { - Tls::destroy(AbiKey::from_usize(key)) -} diff --git a/library/std/src/sys/pal/solid/mod.rs b/library/std/src/sys/pal/solid/mod.rs index 3f6ff37903a..9a7741ddda7 100644 --- a/library/std/src/sys/pal/solid/mod.rs +++ b/library/std/src/sys/pal/solid/mod.rs @@ -33,8 +33,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub use self::itron::thread; -pub mod thread_local_dtor; -pub mod thread_local_key; pub use self::itron::thread_parking; pub mod time; diff --git a/library/std/src/sys/pal/solid/thread_local_dtor.rs b/library/std/src/sys/pal/solid/thread_local_dtor.rs deleted file mode 100644 index 26918a4fcb0..00000000000 --- a/library/std/src/sys/pal/solid/thread_local_dtor.rs +++ /dev/null @@ -1,43 +0,0 @@ -#![cfg(target_thread_local)] -#![unstable(feature = "thread_local_internals", issue = "none")] - -// Simplify dtor registration by using a list of destructors. - -use super::{abi, itron::task}; -use crate::cell::{Cell, RefCell}; - -#[thread_local] -static REGISTERED: Cell<bool> = Cell::new(false); - -#[thread_local] -static DTORS: RefCell<Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>> = RefCell::new(Vec::new()); - -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - if !REGISTERED.get() { - let tid = task::current_task_id_aborting(); - // Register `tls_dtor` to make sure the TLS destructors are called - // for tasks created by other means than `std::thread` - unsafe { abi::SOLID_TLS_AddDestructor(tid as i32, tls_dtor) }; - REGISTERED.set(true); - } - - match DTORS.try_borrow_mut() { - Ok(mut dtors) => dtors.push((t, dtor)), - Err(_) => rtabort!("global allocator may not use TLS"), - } -} - -pub unsafe fn run_dtors() { - let mut list = DTORS.take(); - while !list.is_empty() { - for (ptr, dtor) in list { - unsafe { dtor(ptr) }; - } - - list = DTORS.take(); - } -} - -unsafe extern "C" fn tls_dtor(_unused: *mut u8) { - unsafe { run_dtors() }; -} diff --git a/library/std/src/sys/pal/solid/thread_local_key.rs b/library/std/src/sys/pal/solid/thread_local_key.rs deleted file mode 100644 index b37bf999698..00000000000 --- a/library/std/src/sys/pal/solid/thread_local_key.rs +++ /dev/null @@ -1,21 +0,0 @@ -pub type Key = usize; - -#[inline] -pub unsafe fn create(_dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key { - panic!("should not be used on the solid target"); -} - -#[inline] -pub unsafe fn set(_key: Key, _value: *mut u8) { - panic!("should not be used on the solid target"); -} - -#[inline] -pub unsafe fn get(_key: Key) -> *mut u8 { - panic!("should not be used on the solid target"); -} - -#[inline] -pub unsafe fn destroy(_key: Key) { - panic!("should not be used on the solid target"); -} diff --git a/library/std/src/sys/pal/teeos/mod.rs b/library/std/src/sys/pal/teeos/mod.rs index 6dd465a12ed..2a789e72722 100644 --- a/library/std/src/sys/pal/teeos/mod.rs +++ b/library/std/src/sys/pal/teeos/mod.rs @@ -27,9 +27,6 @@ pub mod process; mod rand; pub mod stdio; pub mod thread; -pub mod thread_local_dtor; -#[path = "../unix/thread_local_key.rs"] -pub mod thread_local_key; #[allow(non_upper_case_globals)] #[path = "../unix/time.rs"] pub mod time; diff --git a/library/std/src/sys/pal/teeos/thread_local_dtor.rs b/library/std/src/sys/pal/teeos/thread_local_dtor.rs deleted file mode 100644 index 5c6bc4d6750..00000000000 --- a/library/std/src/sys/pal/teeos/thread_local_dtor.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - use crate::sys_common::thread_local_dtor::register_dtor_fallback; - register_dtor_fallback(t, dtor); -} diff --git a/library/std/src/sys/pal/uefi/mod.rs b/library/std/src/sys/pal/uefi/mod.rs index 48b74df1384..408031a4616 100644 --- a/library/std/src/sys/pal/uefi/mod.rs +++ b/library/std/src/sys/pal/uefi/mod.rs @@ -28,8 +28,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -#[path = "../unsupported/thread_local_key.rs"] -pub mod thread_local_key; pub mod time; mod helpers; diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index b370f06e92b..16fc2011d70 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -33,8 +33,6 @@ pub mod rand; pub mod stack_overflow; pub mod stdio; pub mod thread; -pub mod thread_local_dtor; -pub mod thread_local_key; pub mod thread_parking; pub mod time; diff --git a/library/std/src/sys/pal/unix/thread_local_dtor.rs b/library/std/src/sys/pal/unix/thread_local_dtor.rs deleted file mode 100644 index 75db6e112ed..00000000000 --- a/library/std/src/sys/pal/unix/thread_local_dtor.rs +++ /dev/null @@ -1,126 +0,0 @@ -#![cfg(target_thread_local)] -#![unstable(feature = "thread_local_internals", issue = "none")] - -//! Provides thread-local destructors without an associated "key", which -//! can be more efficient. - -// Since what appears to be glibc 2.18 this symbol has been shipped which -// GCC and clang both use to invoke destructors in thread_local globals, so -// let's do the same! -// -// Note, however, that we run on lots older linuxes, as well as cross -// compiling from a newer linux to an older linux, so we also have a -// fallback implementation to use as well. -#[cfg(any( - target_os = "linux", - target_os = "android", - target_os = "fuchsia", - target_os = "redox", - target_os = "hurd", - target_os = "netbsd", - target_os = "dragonfly" -))] -// FIXME: The Rust compiler currently omits weakly function definitions (i.e., -// __cxa_thread_atexit_impl) and its metadata from LLVM IR. -#[no_sanitize(cfi, kcfi)] -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - use crate::mem; - use crate::sys_common::thread_local_dtor::register_dtor_fallback; - - /// This is necessary because the __cxa_thread_atexit_impl implementation - /// std links to by default may be a C or C++ implementation that was not - /// compiled using the Clang integer normalization option. - #[cfg(sanitizer_cfi_normalize_integers)] - use core::ffi::c_int; - #[cfg(not(sanitizer_cfi_normalize_integers))] - #[cfi_encoding = "i"] - #[repr(transparent)] - pub struct c_int(#[allow(dead_code)] pub libc::c_int); - - extern "C" { - #[linkage = "extern_weak"] - static __dso_handle: *mut u8; - #[linkage = "extern_weak"] - static __cxa_thread_atexit_impl: Option< - extern "C" fn( - unsafe extern "C" fn(*mut libc::c_void), - *mut libc::c_void, - *mut libc::c_void, - ) -> c_int, - >; - } - - if let Some(f) = __cxa_thread_atexit_impl { - unsafe { - f( - mem::transmute::< - unsafe extern "C" fn(*mut u8), - unsafe extern "C" fn(*mut libc::c_void), - >(dtor), - t.cast(), - core::ptr::addr_of!(__dso_handle) as *mut _, - ); - } - return; - } - register_dtor_fallback(t, dtor); -} - -// This implementation is very similar to register_dtor_fallback in -// sys_common/thread_local.rs. The main difference is that we want to hook into -// macOS's analog of the above linux function, _tlv_atexit. OSX will run the -// registered dtors before any TLS slots get freed, and when the main thread -// exits. -// -// Unfortunately, calling _tlv_atexit while tls dtors are running is UB. The -// workaround below is to register, via _tlv_atexit, a custom DTOR list once per -// thread. thread_local dtors are pushed to the DTOR list without calling -// _tlv_atexit. -#[cfg(target_vendor = "apple")] -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - use crate::cell::{Cell, RefCell}; - use crate::ptr; - - #[thread_local] - static REGISTERED: Cell<bool> = Cell::new(false); - - #[thread_local] - static DTORS: RefCell<Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>> = RefCell::new(Vec::new()); - - if !REGISTERED.get() { - _tlv_atexit(run_dtors, ptr::null_mut()); - REGISTERED.set(true); - } - - extern "C" { - fn _tlv_atexit(dtor: unsafe extern "C" fn(*mut u8), arg: *mut u8); - } - - match DTORS.try_borrow_mut() { - Ok(mut dtors) => dtors.push((t, dtor)), - Err(_) => rtabort!("global allocator may not use TLS"), - } - - unsafe extern "C" fn run_dtors(_: *mut u8) { - let mut list = DTORS.take(); - while !list.is_empty() { - for (ptr, dtor) in list { - dtor(ptr); - } - list = DTORS.take(); - } - } -} - -#[cfg(any( - target_os = "vxworks", - target_os = "horizon", - target_os = "emscripten", - target_os = "aix", - target_os = "freebsd", -))] -#[cfg_attr(target_family = "wasm", allow(unused))] // might remain unused depending on target details (e.g. wasm32-unknown-emscripten) -pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - use crate::sys_common::thread_local_dtor::register_dtor_fallback; - register_dtor_fallback(t, dtor); -} diff --git a/library/std/src/sys/pal/unix/thread_local_key.rs b/library/std/src/sys/pal/unix/thread_local_key.rs deleted file mode 100644 index 2b2d079ee4d..00000000000 --- a/library/std/src/sys/pal/unix/thread_local_key.rs +++ /dev/null @@ -1,29 +0,0 @@ -#![allow(dead_code)] // not used on all platforms - -use crate::mem; - -pub type Key = libc::pthread_key_t; - -#[inline] -pub unsafe fn create(dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key { - let mut key = 0; - assert_eq!(libc::pthread_key_create(&mut key, mem::transmute(dtor)), 0); - key -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - let r = libc::pthread_setspecific(key, value as *mut _); - debug_assert_eq!(r, 0); -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - libc::pthread_getspecific(key) as *mut u8 -} - -#[inline] -pub unsafe fn destroy(key: Key) { - let r = libc::pthread_key_delete(key); - debug_assert_eq!(r, 0); -} diff --git a/library/std/src/sys/pal/unsupported/mod.rs b/library/std/src/sys/pal/unsupported/mod.rs index 01f5cfd4297..442e6042ad5 100644 --- a/library/std/src/sys/pal/unsupported/mod.rs +++ b/library/std/src/sys/pal/unsupported/mod.rs @@ -11,9 +11,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -#[cfg(target_thread_local)] -pub mod thread_local_dtor; -pub mod thread_local_key; pub mod time; mod common; diff --git a/library/std/src/sys/pal/unsupported/thread_local_dtor.rs b/library/std/src/sys/pal/unsupported/thread_local_dtor.rs deleted file mode 100644 index 84660ea5881..00000000000 --- a/library/std/src/sys/pal/unsupported/thread_local_dtor.rs +++ /dev/null @@ -1,10 +0,0 @@ -#![unstable(feature = "thread_local_internals", issue = "none")] - -#[cfg_attr(target_family = "wasm", allow(unused))] // unused on wasm32-unknown-unknown -pub unsafe fn register_dtor(_t: *mut u8, _dtor: unsafe extern "C" fn(*mut u8)) { - // FIXME: right now there is no concept of "thread exit", but this is likely - // going to show up at some point in the form of an exported symbol that the - // wasm runtime is going to be expected to call. For now we basically just - // ignore the arguments, but if such a function starts to exist it will - // likely look like the OSX implementation in `unix/fast_thread_local.rs` -} diff --git a/library/std/src/sys/pal/unsupported/thread_local_key.rs b/library/std/src/sys/pal/unsupported/thread_local_key.rs deleted file mode 100644 index b6e5e4cd2e1..00000000000 --- a/library/std/src/sys/pal/unsupported/thread_local_key.rs +++ /dev/null @@ -1,21 +0,0 @@ -pub type Key = usize; - -#[inline] -pub unsafe fn create(_dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key { - panic!("should not be used on this target"); -} - -#[inline] -pub unsafe fn set(_key: Key, _value: *mut u8) { - panic!("should not be used on this target"); -} - -#[inline] -pub unsafe fn get(_key: Key) -> *mut u8 { - panic!("should not be used on this target"); -} - -#[inline] -pub unsafe fn destroy(_key: Key) { - panic!("should not be used on this target"); -} diff --git a/library/std/src/sys/pal/wasi/mod.rs b/library/std/src/sys/pal/wasi/mod.rs index c1266619b36..8dfb733043e 100644 --- a/library/std/src/sys/pal/wasi/mod.rs +++ b/library/std/src/sys/pal/wasi/mod.rs @@ -33,10 +33,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -#[path = "../unsupported/thread_local_dtor.rs"] -pub mod thread_local_dtor; -#[path = "../unsupported/thread_local_key.rs"] -pub mod thread_local_key; pub mod time; #[path = "../unsupported/common.rs"] diff --git a/library/std/src/sys/pal/wasip2/mod.rs b/library/std/src/sys/pal/wasip2/mod.rs index 6787ffb4bed..7af0917b8ed 100644 --- a/library/std/src/sys/pal/wasip2/mod.rs +++ b/library/std/src/sys/pal/wasip2/mod.rs @@ -34,10 +34,6 @@ pub mod process; pub mod stdio; #[path = "../wasi/thread.rs"] pub mod thread; -#[path = "../unsupported/thread_local_dtor.rs"] -pub mod thread_local_dtor; -#[path = "../unsupported/thread_local_key.rs"] -pub mod thread_local_key; #[path = "../wasi/time.rs"] pub mod time; diff --git a/library/std/src/sys/pal/wasm/mod.rs b/library/std/src/sys/pal/wasm/mod.rs index 75dd10826cc..4c34859e918 100644 --- a/library/std/src/sys/pal/wasm/mod.rs +++ b/library/std/src/sys/pal/wasm/mod.rs @@ -34,10 +34,6 @@ pub mod pipe; pub mod process; #[path = "../unsupported/stdio.rs"] pub mod stdio; -#[path = "../unsupported/thread_local_dtor.rs"] -pub mod thread_local_dtor; -#[path = "../unsupported/thread_local_key.rs"] -pub mod thread_local_key; #[path = "../unsupported/time.rs"] pub mod time; diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index 9d58ce05f01..6ec82693077 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -54,6 +54,7 @@ pub const EXIT_FAILURE: u32 = 1; pub const CONDITION_VARIABLE_INIT: CONDITION_VARIABLE = CONDITION_VARIABLE { Ptr: ptr::null_mut() }; #[cfg(target_vendor = "win7")] pub const SRWLOCK_INIT: SRWLOCK = SRWLOCK { Ptr: ptr::null_mut() }; +#[cfg(not(target_thread_local))] pub const INIT_ONCE_STATIC_INIT: INIT_ONCE = INIT_ONCE { Ptr: ptr::null_mut() }; // Some windows_sys types have different signs than the types we use. diff --git a/library/std/src/sys/pal/windows/mod.rs b/library/std/src/sys/pal/windows/mod.rs index 402a205977b..a1bc2965e2e 100644 --- a/library/std/src/sys/pal/windows/mod.rs +++ b/library/std/src/sys/pal/windows/mod.rs @@ -31,8 +31,6 @@ pub mod process; pub mod rand; pub mod stdio; pub mod thread; -pub mod thread_local_dtor; -pub mod thread_local_key; pub mod time; cfg_if::cfg_if! { if #[cfg(not(target_vendor = "uwp"))] { diff --git a/library/std/src/sys/pal/windows/thread_local_dtor.rs b/library/std/src/sys/pal/windows/thread_local_dtor.rs deleted file mode 100644 index cf542d2bfb8..00000000000 --- a/library/std/src/sys/pal/windows/thread_local_dtor.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Implements thread-local destructors that are not associated with any -//! particular data. - -#![unstable(feature = "thread_local_internals", issue = "none")] -#![cfg(target_thread_local)] - -pub use super::thread_local_key::register_keyless_dtor as register_dtor; diff --git a/library/std/src/sys/pal/windows/thread_local_key.rs b/library/std/src/sys/pal/windows/thread_local_key.rs deleted file mode 100644 index e5ba619fc6b..00000000000 --- a/library/std/src/sys/pal/windows/thread_local_key.rs +++ /dev/null @@ -1,351 +0,0 @@ -use crate::cell::UnsafeCell; -use crate::ptr; -use crate::sync::atomic::{ - AtomicPtr, AtomicU32, - Ordering::{AcqRel, Acquire, Relaxed, Release}, -}; -use crate::sys::c; - -#[cfg(test)] -mod tests; - -// Using a per-thread list avoids the problems in synchronizing global state. -#[thread_local] -#[cfg(target_thread_local)] -static DESTRUCTORS: crate::cell::RefCell<Vec<(*mut u8, unsafe extern "C" fn(*mut u8))>> = - crate::cell::RefCell::new(Vec::new()); - -// Ensure this can never be inlined because otherwise this may break in dylibs. -// See #44391. -#[inline(never)] -#[cfg(target_thread_local)] -pub unsafe fn register_keyless_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - dtors_used(); - match DESTRUCTORS.try_borrow_mut() { - Ok(mut dtors) => dtors.push((t, dtor)), - Err(_) => rtabort!("global allocator may not use TLS"), - } -} - -#[inline(never)] // See comment above -#[cfg(target_thread_local)] -/// Runs destructors. This should not be called until thread exit. -unsafe fn run_keyless_dtors() { - // Drop all the destructors. - // - // Note: While this is potentially an infinite loop, it *should* be - // the case that this loop always terminates because we provide the - // guarantee that a TLS key cannot be set after it is flagged for - // destruction. - loop { - // Use a let-else binding to ensure the `RefCell` guard is dropped - // immediately. Otherwise, a panic would occur if a TLS destructor - // tries to access the list. - let Some((ptr, dtor)) = DESTRUCTORS.borrow_mut().pop() else { - break; - }; - (dtor)(ptr); - } - // We're done so free the memory. - DESTRUCTORS.replace(Vec::new()); -} - -type Key = c::DWORD; -type Dtor = unsafe extern "C" fn(*mut u8); - -// Turns out, like pretty much everything, Windows is pretty close the -// functionality that Unix provides, but slightly different! In the case of -// TLS, Windows does not provide an API to provide a destructor for a TLS -// variable. This ends up being pretty crucial to this implementation, so we -// need a way around this. -// -// The solution here ended up being a little obscure, but fear not, the -// internet has informed me [1][2] that this solution is not unique (no way -// I could have thought of it as well!). The key idea is to insert some hook -// somewhere to run arbitrary code on thread termination. With this in place -// we'll be able to run anything we like, including all TLS destructors! -// -// To accomplish this feat, we perform a number of threads, all contained -// within this module: -// -// * All TLS destructors are tracked by *us*, not the Windows runtime. This -// means that we have a global list of destructors for each TLS key that -// we know about. -// * When a thread exits, we run over the entire list and run dtors for all -// non-null keys. This attempts to match Unix semantics in this regard. -// -// For more details and nitty-gritty, see the code sections below! -// -// [1]: https://www.codeproject.com/Articles/8113/Thread-Local-Storage-The-C-Way -// [2]: https://github.com/ChromiumWebApps/chromium/blob/master/base/threading/thread_local_storage_win.cc#L42 - -pub struct StaticKey { - /// The key value shifted up by one. Since TLS_OUT_OF_INDEXES == DWORD::MAX - /// is not a valid key value, this allows us to use zero as sentinel value - /// without risking overflow. - key: AtomicU32, - dtor: Option<Dtor>, - next: AtomicPtr<StaticKey>, - /// Currently, destructors cannot be unregistered, so we cannot use racy - /// initialization for keys. Instead, we need synchronize initialization. - /// Use the Windows-provided `Once` since it does not require TLS. - once: UnsafeCell<c::INIT_ONCE>, -} - -impl StaticKey { - #[inline] - pub const fn new(dtor: Option<Dtor>) -> StaticKey { - StaticKey { - key: AtomicU32::new(0), - dtor, - next: AtomicPtr::new(ptr::null_mut()), - once: UnsafeCell::new(c::INIT_ONCE_STATIC_INIT), - } - } - - #[inline] - pub unsafe fn set(&'static self, val: *mut u8) { - let r = c::TlsSetValue(self.key(), val.cast()); - debug_assert_eq!(r, c::TRUE); - } - - #[inline] - pub unsafe fn get(&'static self) -> *mut u8 { - c::TlsGetValue(self.key()).cast() - } - - #[inline] - unsafe fn key(&'static self) -> Key { - match self.key.load(Acquire) { - 0 => self.init(), - key => key - 1, - } - } - - #[cold] - unsafe fn init(&'static self) -> Key { - if self.dtor.is_some() { - dtors_used(); - let mut pending = c::FALSE; - let r = c::InitOnceBeginInitialize(self.once.get(), 0, &mut pending, ptr::null_mut()); - assert_eq!(r, c::TRUE); - - if pending == c::FALSE { - // Some other thread initialized the key, load it. - self.key.load(Relaxed) - 1 - } else { - let key = c::TlsAlloc(); - if key == c::TLS_OUT_OF_INDEXES { - // Wakeup the waiting threads before panicking to avoid deadlock. - c::InitOnceComplete(self.once.get(), c::INIT_ONCE_INIT_FAILED, ptr::null_mut()); - panic!("out of TLS indexes"); - } - - register_dtor(self); - - // Release-storing the key needs to be the last thing we do. - // This is because in `fn key()`, other threads will do an acquire load of the key, - // and if that sees this write then it will entirely bypass the `InitOnce`. We thus - // need to establish synchronization through `key`. In particular that acquire load - // must happen-after the register_dtor above, to ensure the dtor actually runs! - self.key.store(key + 1, Release); - - let r = c::InitOnceComplete(self.once.get(), 0, ptr::null_mut()); - debug_assert_eq!(r, c::TRUE); - - key - } - } else { - // If there is no destructor to clean up, we can use racy initialization. - - let key = c::TlsAlloc(); - assert_ne!(key, c::TLS_OUT_OF_INDEXES, "out of TLS indexes"); - - match self.key.compare_exchange(0, key + 1, AcqRel, Acquire) { - Ok(_) => key, - Err(new) => { - // Some other thread completed initialization first, so destroy - // our key and use theirs. - let r = c::TlsFree(key); - debug_assert_eq!(r, c::TRUE); - new - 1 - } - } - } - } -} - -unsafe impl Send for StaticKey {} -unsafe impl Sync for StaticKey {} - -// ------------------------------------------------------------------------- -// Dtor registration -// -// Windows has no native support for running destructors so we manage our own -// list of destructors to keep track of how to destroy keys. We then install a -// callback later to get invoked whenever a thread exits, running all -// appropriate destructors. -// -// Currently unregistration from this list is not supported. A destructor can be -// registered but cannot be unregistered. There's various simplifying reasons -// for doing this, the big ones being: -// -// 1. Currently we don't even support deallocating TLS keys, so normal operation -// doesn't need to deallocate a destructor. -// 2. There is no point in time where we know we can unregister a destructor -// because it could always be getting run by some remote thread. -// -// Typically processes have a statically known set of TLS keys which is pretty -// small, and we'd want to keep this memory alive for the whole process anyway -// really. - -static DTORS: AtomicPtr<StaticKey> = AtomicPtr::new(ptr::null_mut()); - -/// Should only be called once per key, otherwise loops or breaks may occur in -/// the linked list. -unsafe fn register_dtor(key: &'static StaticKey) { - // Ensure this is never run when native thread locals are available. - assert_eq!(false, cfg!(target_thread_local)); - let this = <*const StaticKey>::cast_mut(key); - // Use acquire ordering to pass along the changes done by the previously - // registered keys when we store the new head with release ordering. - let mut head = DTORS.load(Acquire); - loop { - key.next.store(head, Relaxed); - match DTORS.compare_exchange_weak(head, this, Release, Acquire) { - Ok(_) => break, - Err(new) => head = new, - } - } -} - -// ------------------------------------------------------------------------- -// Where the Magic (TM) Happens -// -// If you're looking at this code, and wondering "what is this doing?", -// you're not alone! I'll try to break this down step by step: -// -// # What's up with CRT$XLB? -// -// For anything about TLS destructors to work on Windows, we have to be able -// to run *something* when a thread exits. To do so, we place a very special -// static in a very special location. If this is encoded in just the right -// way, the kernel's loader is apparently nice enough to run some function -// of ours whenever a thread exits! How nice of the kernel! -// -// Lots of detailed information can be found in source [1] above, but the -// gist of it is that this is leveraging a feature of Microsoft's PE format -// (executable format) which is not actually used by any compilers today. -// This apparently translates to any callbacks in the ".CRT$XLB" section -// being run on certain events. -// -// So after all that, we use the compiler's #[link_section] feature to place -// a callback pointer into the magic section so it ends up being called. -// -// # What's up with this callback? -// -// The callback specified receives a number of parameters from... someone! -// (the kernel? the runtime? I'm not quite sure!) There are a few events that -// this gets invoked for, but we're currently only interested on when a -// thread or a process "detaches" (exits). The process part happens for the -// last thread and the thread part happens for any normal thread. -// -// # Ok, what's up with running all these destructors? -// -// This will likely need to be improved over time, but this function -// attempts a "poor man's" destructor callback system. Once we've got a list -// of what to run, we iterate over all keys, check their values, and then run -// destructors if the values turn out to be non null (setting them to null just -// beforehand). We do this a few times in a loop to basically match Unix -// semantics. If we don't reach a fixed point after a short while then we just -// inevitably leak something most likely. -// -// # The article mentions weird stuff about "/INCLUDE"? -// -// It sure does! Specifically we're talking about this quote: -// -// The Microsoft run-time library facilitates this process by defining a -// memory image of the TLS Directory and giving it the special name -// “__tls_used” (Intel x86 platforms) or “_tls_used” (other platforms). The -// linker looks for this memory image and uses the data there to create the -// TLS Directory. Other compilers that support TLS and work with the -// Microsoft linker must use this same technique. -// -// Basically what this means is that if we want support for our TLS -// destructors/our hook being called then we need to make sure the linker does -// not omit this symbol. Otherwise it will omit it and our callback won't be -// wired up. -// -// We don't actually use the `/INCLUDE` linker flag here like the article -// mentions because the Rust compiler doesn't propagate linker flags, but -// instead we use a shim function which performs a volatile 1-byte load from -// the address of the symbol to ensure it sticks around. - -#[link_section = ".CRT$XLB"] -#[cfg_attr(miri, used)] // Miri only considers explicitly `#[used]` statics for `lookup_link_section` -pub static p_thread_callback: unsafe extern "system" fn(c::LPVOID, c::DWORD, c::LPVOID) = - on_tls_callback; - -fn dtors_used() { - // we don't want LLVM eliminating p_thread_callback when destructors are used. - // when the symbol makes it to the linker the linker will take over - unsafe { crate::intrinsics::volatile_load(&p_thread_callback) }; -} - -unsafe extern "system" fn on_tls_callback(_h: c::LPVOID, dwReason: c::DWORD, _pv: c::LPVOID) { - if dwReason == c::DLL_THREAD_DETACH || dwReason == c::DLL_PROCESS_DETACH { - #[cfg(not(target_thread_local))] - run_dtors(); - #[cfg(target_thread_local)] - run_keyless_dtors(); - } - - // See comments above for what this is doing. Note that we don't need this - // trickery on GNU windows, just on MSVC. - #[cfg(all(target_env = "msvc", not(target_thread_local)))] - { - extern "C" { - static _tls_used: u8; - } - crate::intrinsics::volatile_load(&_tls_used); - } -} - -#[cfg(not(target_thread_local))] -unsafe fn run_dtors() { - for _ in 0..5 { - let mut any_run = false; - - // Use acquire ordering to observe key initialization. - let mut cur = DTORS.load(Acquire); - while !cur.is_null() { - let pre_key = (*cur).key.load(Acquire); - let dtor = (*cur).dtor.unwrap(); - cur = (*cur).next.load(Relaxed); - - // In StaticKey::init, we register the dtor before setting `key`. - // So if one thread's `run_dtors` races with another thread executing `init` on the same - // `StaticKey`, we can encounter a key of 0 here. That means this key was never - // initialized in this thread so we can safely skip it. - if pre_key == 0 { - continue; - } - // If this is non-zero, then via the `Acquire` load above we synchronized with - // everything relevant for this key. (It's not clear that this is needed, since the - // release-acquire pair on DTORS also establishes synchronization, but better safe than - // sorry.) - let key = pre_key - 1; - - let ptr = c::TlsGetValue(key); - if !ptr.is_null() { - c::TlsSetValue(key, ptr::null_mut()); - dtor(ptr as *mut _); - any_run = true; - } - } - - if !any_run { - break; - } - } -} diff --git a/library/std/src/sys/pal/windows/thread_local_key/tests.rs b/library/std/src/sys/pal/windows/thread_local_key/tests.rs deleted file mode 100644 index 4119f990968..00000000000 --- a/library/std/src/sys/pal/windows/thread_local_key/tests.rs +++ /dev/null @@ -1,57 +0,0 @@ -// This file only tests the thread local key fallback. -// Windows targets with native thread local support do not use this. -#![cfg(not(target_thread_local))] - -use super::StaticKey; -use crate::ptr; - -#[test] -fn smoke() { - static K1: StaticKey = StaticKey::new(None); - static K2: StaticKey = StaticKey::new(None); - - unsafe { - assert!(K1.get().is_null()); - assert!(K2.get().is_null()); - K1.set(ptr::without_provenance_mut(1)); - K2.set(ptr::without_provenance_mut(2)); - assert_eq!(K1.get() as usize, 1); - assert_eq!(K2.get() as usize, 2); - } -} - -#[test] -fn destructors() { - use crate::mem::ManuallyDrop; - use crate::sync::Arc; - use crate::thread; - - unsafe extern "C" fn destruct(ptr: *mut u8) { - drop(Arc::from_raw(ptr as *const ())); - } - - static KEY: StaticKey = StaticKey::new(Some(destruct)); - - let shared1 = Arc::new(()); - let shared2 = Arc::clone(&shared1); - - unsafe { - assert!(KEY.get().is_null()); - KEY.set(Arc::into_raw(shared1) as *mut u8); - } - - thread::spawn(move || unsafe { - assert!(KEY.get().is_null()); - KEY.set(Arc::into_raw(shared2) as *mut u8); - }) - .join() - .unwrap(); - - // Leak the Arc, let the TLS destructor clean it up. - let shared1 = unsafe { ManuallyDrop::new(Arc::from_raw(KEY.get() as *const ())) }; - assert_eq!( - Arc::strong_count(&shared1), - 1, - "destructor should have dropped the other reference on thread exit" - ); -} diff --git a/library/std/src/sys/pal/xous/mod.rs b/library/std/src/sys/pal/xous/mod.rs index 68189bcc2e3..a28a52e305e 100644 --- a/library/std/src/sys/pal/xous/mod.rs +++ b/library/std/src/sys/pal/xous/mod.rs @@ -17,7 +17,6 @@ pub mod pipe; pub mod process; pub mod stdio; pub mod thread; -pub mod thread_local_key; pub mod time; #[path = "../unsupported/common.rs"] diff --git a/library/std/src/sys/pal/xous/thread.rs b/library/std/src/sys/pal/xous/thread.rs index da7d722cc70..279f24f9ee8 100644 --- a/library/std/src/sys/pal/xous/thread.rs +++ b/library/std/src/sys/pal/xous/thread.rs @@ -81,7 +81,7 @@ impl Thread { // Destroy TLS, which will free the TLS page and call the destructor for // any thread local storage (if any). unsafe { - crate::sys::thread_local_key::destroy_tls(); + crate::sys::thread_local::key::destroy_tls(); } // Deallocate the stack memory, along with the guard pages. Afterwards, diff --git a/library/std/src/sys/pal/xous/thread_local_key.rs b/library/std/src/sys/pal/xous/thread_local_key.rs deleted file mode 100644 index 6c29813c79d..00000000000 --- a/library/std/src/sys/pal/xous/thread_local_key.rs +++ /dev/null @@ -1,215 +0,0 @@ -use crate::mem::ManuallyDrop; -use crate::ptr; -use crate::sync::atomic::AtomicPtr; -use crate::sync::atomic::AtomicUsize; -use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release}; -use core::arch::asm; - -use crate::os::xous::ffi::{map_memory, unmap_memory, MemoryFlags}; - -/// Thread Local Storage -/// -/// Currently, we are limited to 1023 TLS entries. The entries -/// live in a page of memory that's unique per-process, and is -/// stored in the `$tp` register. If this register is 0, then -/// TLS has not been initialized and thread cleanup can be skipped. -/// -/// The index into this register is the `key`. This key is identical -/// between all threads, but indexes a different offset within this -/// pointer. -pub type Key = usize; - -pub type Dtor = unsafe extern "C" fn(*mut u8); - -const TLS_MEMORY_SIZE: usize = 4096; - -/// TLS keys start at `1`. Index `0` is unused -#[cfg(not(test))] -#[export_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key13TLS_KEY_INDEXE"] -static TLS_KEY_INDEX: AtomicUsize = AtomicUsize::new(1); - -#[cfg(not(test))] -#[export_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key9DTORSE"] -static DTORS: AtomicPtr<Node> = AtomicPtr::new(ptr::null_mut()); - -#[cfg(test)] -extern "Rust" { - #[link_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key13TLS_KEY_INDEXE"] - static TLS_KEY_INDEX: AtomicUsize; - - #[link_name = "_ZN16__rust_internals3std3sys4xous16thread_local_key9DTORSE"] - static DTORS: AtomicPtr<Node>; -} - -fn tls_ptr_addr() -> *mut *mut u8 { - let mut tp: usize; - unsafe { - asm!( - "mv {}, tp", - out(reg) tp, - ); - } - core::ptr::with_exposed_provenance_mut::<*mut u8>(tp) -} - -/// Create an area of memory that's unique per thread. This area will -/// contain all thread local pointers. -fn tls_table() -> &'static mut [*mut u8] { - let tp = tls_ptr_addr(); - - if !tp.is_null() { - return unsafe { - core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / core::mem::size_of::<*mut u8>()) - }; - } - // If the TP register is `0`, then this thread hasn't initialized - // its TLS yet. Allocate a new page to store this memory. - let tp = unsafe { - map_memory( - None, - None, - TLS_MEMORY_SIZE / core::mem::size_of::<*mut u8>(), - MemoryFlags::R | MemoryFlags::W, - ) - .expect("Unable to allocate memory for thread local storage") - }; - - for val in tp.iter() { - assert!(*val as usize == 0); - } - - unsafe { - // Set the thread's `$tp` register - asm!( - "mv tp, {}", - in(reg) tp.as_mut_ptr() as usize, - ); - } - tp -} - -#[inline] -pub unsafe fn create(dtor: Option<Dtor>) -> Key { - // Allocate a new TLS key. These keys are shared among all threads. - #[allow(unused_unsafe)] - let key = unsafe { TLS_KEY_INDEX.fetch_add(1, Relaxed) }; - if let Some(f) = dtor { - unsafe { register_dtor(key, f) }; - } - key -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - assert!((key < 1022) && (key >= 1)); - let table = tls_table(); - table[key] = value; -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - assert!((key < 1022) && (key >= 1)); - tls_table()[key] -} - -#[inline] -pub unsafe fn destroy(_key: Key) { - // Just leak the key. Probably not great on long-running systems that create - // lots of TLS variables, but in practice that's not an issue. -} - -// ------------------------------------------------------------------------- -// Dtor registration (stolen from Windows) -// -// Xous has no native support for running destructors so we manage our own -// list of destructors to keep track of how to destroy keys. We then install a -// callback later to get invoked whenever a thread exits, running all -// appropriate destructors. -// -// Currently unregistration from this list is not supported. A destructor can be -// registered but cannot be unregistered. There's various simplifying reasons -// for doing this, the big ones being: -// -// 1. Currently we don't even support deallocating TLS keys, so normal operation -// doesn't need to deallocate a destructor. -// 2. There is no point in time where we know we can unregister a destructor -// because it could always be getting run by some remote thread. -// -// Typically processes have a statically known set of TLS keys which is pretty -// small, and we'd want to keep this memory alive for the whole process anyway -// really. -// -// Perhaps one day we can fold the `Box` here into a static allocation, -// expanding the `StaticKey` structure to contain not only a slot for the TLS -// key but also a slot for the destructor queue on windows. An optimization for -// another day! - -struct Node { - dtor: Dtor, - key: Key, - next: *mut Node, -} - -unsafe fn register_dtor(key: Key, dtor: Dtor) { - let mut node = ManuallyDrop::new(Box::new(Node { key, dtor, next: ptr::null_mut() })); - - #[allow(unused_unsafe)] - let mut head = unsafe { DTORS.load(Acquire) }; - loop { - node.next = head; - #[allow(unused_unsafe)] - match unsafe { DTORS.compare_exchange(head, &mut **node, Release, Acquire) } { - Ok(_) => return, // nothing to drop, we successfully added the node to the list - Err(cur) => head = cur, - } - } -} - -pub unsafe fn destroy_tls() { - let tp = tls_ptr_addr(); - - // If the pointer address is 0, then this thread has no TLS. - if tp.is_null() { - return; - } - - unsafe { run_dtors() }; - - // Finally, free the TLS array - unsafe { - unmap_memory(core::slice::from_raw_parts_mut( - tp, - TLS_MEMORY_SIZE / core::mem::size_of::<usize>(), - )) - .unwrap() - }; -} - -unsafe fn run_dtors() { - let mut any_run = true; - - // Run the destructor "some" number of times. This is 5x on Windows, - // so we copy it here. This allows TLS variables to create new - // TLS variables upon destruction that will also get destroyed. - // Keep going until we run out of tries or until we have nothing - // left to destroy. - for _ in 0..5 { - if !any_run { - break; - } - any_run = false; - #[allow(unused_unsafe)] - let mut cur = unsafe { DTORS.load(Acquire) }; - while !cur.is_null() { - let ptr = unsafe { get((*cur).key) }; - - if !ptr.is_null() { - unsafe { set((*cur).key, ptr::null_mut()) }; - unsafe { ((*cur).dtor)(ptr as *mut _) }; - any_run = true; - } - - unsafe { cur = (*cur).next }; - } - } -} diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs index 0b22eabca6d..bacde9d880c 100644 --- a/library/std/src/sys/pal/zkvm/mod.rs +++ b/library/std/src/sys/pal/zkvm/mod.rs @@ -25,7 +25,6 @@ pub mod pipe; #[path = "../unsupported/process.rs"] pub mod process; pub mod stdio; -pub mod thread_local_key; #[path = "../unsupported/time.rs"] pub mod time; diff --git a/library/std/src/sys/pal/zkvm/thread_local_key.rs b/library/std/src/sys/pal/zkvm/thread_local_key.rs deleted file mode 100644 index 2f67924c618..00000000000 --- a/library/std/src/sys/pal/zkvm/thread_local_key.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::alloc::{alloc, Layout}; - -pub type Key = usize; - -#[inline] -pub unsafe fn create(_dtor: Option<unsafe extern "C" fn(*mut u8)>) -> Key { - alloc(Layout::new::<*mut u8>()) as _ -} - -#[inline] -pub unsafe fn set(key: Key, value: *mut u8) { - let key: *mut *mut u8 = core::ptr::with_exposed_provenance_mut(key); - *key = value; -} - -#[inline] -pub unsafe fn get(key: Key) -> *mut u8 { - let key: *mut *mut u8 = core::ptr::with_exposed_provenance_mut(key); - *key -} - -#[inline] -pub unsafe fn destroy(_key: Key) {} |
