std: make `thread::current` available in all `thread_local!` destructors

3 files changed, 319 insertions, 94 deletions

diff --git a/library/std/src/thread/current.rs b/library/std/src/thread/current.rs
new file mode 100644
index 00000000000..b38149a0da7
--- /dev/null
+++ b/library/std/src/thread/current.rs
@@ -0,0 +1,252 @@
+use super::{Thread, ThreadId};
+use crate::mem::ManuallyDrop;
+use crate::ptr;
+use crate::sys::thread_local::local_pointer;
+
+const NONE: *mut () = ptr::null_mut();
+const BUSY: *mut () = ptr::without_provenance_mut(1);
+const DESTROYED: *mut () = ptr::without_provenance_mut(2);
+
+local_pointer! {
+    static CURRENT;
+}
+
+/// Persistent storage for the thread ID.
+///
+/// We store the thread ID so that it never gets destroyed during the lifetime
+/// of a thread, either using `#[thread_local]` or multiple `local_pointer!`s.
+mod id {
+    use super::*;
+
+    cfg_if::cfg_if! {
+        if #[cfg(target_thread_local)] {
+            use crate::cell::Cell;
+
+            #[thread_local]
+            static ID: Cell<Option<ThreadId>> = Cell::new(None);
+
+            pub(super) const CHEAP: bool = true;
+
+            pub(super) fn get() -> Option<ThreadId> {
+                ID.get()
+            }
+
+            pub(super) fn set(id: ThreadId) {
+                ID.set(Some(id))
+            }
+        } else if #[cfg(target_pointer_width = "16")] {
+            local_pointer! {
+                static ID0;
+                static ID16;
+                static ID32;
+                static ID48;
+            }
+
+            pub(super) const CHEAP: bool = false;
+
+            pub(super) fn get() -> Option<ThreadId> {
+                let id0 = ID0.get().addr() as u64;
+                let id16 = ID16.get().addr() as u64;
+                let id32 = ID32.get().addr() as u64;
+                let id48 = ID48.get().addr() as u64;
+                ThreadId::from_u64((id48 << 48) + (id32 << 32) + (id16 << 16) + id0)
+            }
+
+            pub(super) fn set(id: ThreadId) {
+                let val = id.as_u64().get();
+                ID0.set(ptr::without_provenance_mut(val as usize));
+                ID16.set(ptr::without_provenance_mut((val >> 16) as usize));
+                ID32.set(ptr::without_provenance_mut((val >> 32) as usize));
+                ID48.set(ptr::without_provenance_mut((val >> 48) as usize));
+            }
+        } else if #[cfg(target_pointer_width = "32")] {
+            local_pointer! {
+                static ID0;
+                static ID32;
+            }
+
+            pub(super) const CHEAP: bool = false;
+
+            pub(super) fn get() -> Option<ThreadId> {
+                let id0 = ID0.get().addr() as u64;
+                let id32 = ID32.get().addr() as u64;
+                ThreadId::from_u64((id32 << 32) + id0)
+            }
+
+            pub(super) fn set(id: ThreadId) {
+                let val = id.as_u64().get();
+                ID0.set(ptr::without_provenance_mut(val as usize));
+                ID32.set(ptr::without_provenance_mut((val >> 32) as usize));
+            }
+        } else {
+            local_pointer! {
+                static ID;
+            }
+
+            pub(super) const CHEAP: bool = true;
+
+            pub(super) fn get() -> Option<ThreadId> {
+                let id = ID.get().addr() as u64;
+                ThreadId::from_u64(id)
+            }
+
+            pub(super) fn set(id: ThreadId) {
+                let val = id.as_u64().get();
+                ID.set(ptr::without_provenance_mut(val as usize));
+            }
+        }
+    }
+
+    #[inline]
+    pub(super) fn get_or_init() -> ThreadId {
+        get().unwrap_or_else(
+            #[cold]
+            || {
+                let id = ThreadId::new();
+                id::set(id);
+                id
+            },
+        )
+    }
+}
+
+/// Sets the thread handle for the current thread.
+///
+/// Aborts if the handle or the ID has been set already.
+pub(crate) fn set_current(thread: Thread) {
+    if CURRENT.get() != NONE || id::get().is_some() {
+        // Using `panic` here can add ~3kB to the binary size. We have complete
+        // control over where this is called, so just abort if there is a bug.
+        rtabort!("thread::set_current should only be called once per thread");
+    }
+
+    id::set(thread.id());
+
+    // Make sure that `crate::rt::thread_cleanup` will be run, which will
+    // call `drop_current`.
+    crate::sys::thread_local::guard::enable();
+    CURRENT.set(thread.into_raw().cast_mut());
+}
+
+/// Gets the id of the thread that invokes it.
+///
+/// This function will always succeed, will always return the same value for
+/// one thread and is guaranteed not to call the global allocator.
+#[inline]
+pub(crate) fn current_id() -> ThreadId {
+    // If accessing the persistant thread ID takes multiple TLS accesses, try
+    // to retrieve it from the current thread handle, which will only take one
+    // TLS access.
+    if !id::CHEAP {
+        let current = CURRENT.get();
+        if current > DESTROYED {
+            unsafe {
+                let current = ManuallyDrop::new(Thread::from_raw(current));
+                return current.id();
+            }
+        }
+    }
+
+    id::get_or_init()
+}
+
+/// Gets a handle to the thread that invokes it, if the handle has been initialized.
+pub(crate) fn try_current() -> Option<Thread> {
+    let current = CURRENT.get();
+    if current > DESTROYED {
+        unsafe {
+            let current = ManuallyDrop::new(Thread::from_raw(current));
+            Some((*current).clone())
+        }
+    } else {
+        None
+    }
+}
+
+/// Gets a handle to the thread that invokes it.
+///
+/// # Examples
+///
+/// Getting a handle to the current thread with `thread::current()`:
+///
+/// ```
+/// use std::thread;
+///
+/// let handler = thread::Builder::new()
+///     .name("named thread".into())
+///     .spawn(|| {
+///         let handle = thread::current();
+///         assert_eq!(handle.name(), Some("named thread"));
+///     })
+///     .unwrap();
+///
+/// handler.join().unwrap();
+/// ```
+#[must_use]
+#[stable(feature = "rust1", since = "1.0.0")]
+pub fn current() -> Thread {
+    let current = CURRENT.get();
+    if current > DESTROYED {
+        unsafe {
+            let current = ManuallyDrop::new(Thread::from_raw(current));
+            (*current).clone()
+        }
+    } else {
+        init_current(current)
+    }
+}
+
+#[cold]
+fn init_current(current: *mut ()) -> Thread {
+    if current == NONE {
+        CURRENT.set(BUSY);
+        // If the thread ID was initialized already, use it.
+        let id = id::get_or_init();
+        let thread = Thread::new_unnamed(id);
+
+        // Make sure that `crate::rt::thread_cleanup` will be run, which will
+        // call `drop_current`.
+        crate::sys::thread_local::guard::enable();
+        CURRENT.set(thread.clone().into_raw().cast_mut());
+        thread
+    } else if current == BUSY {
+        // BUSY exists solely for this check, but as it is in the slow path, the
+        // extra TLS write above shouldn't matter. The alternative is nearly always
+        // a stack overflow.
+
+        // If you came across this message, contact the author of your allocator.
+        // If you are said author: A surprising amount of functions inside the
+        // standard library (e.g. `Mutex`, `thread_local!`, `File` when using long
+        // paths, even `panic!` when using unwinding), need memory allocation, so
+        // you'll get circular dependencies all over the place when using them.
+        // I (joboet) highly recommend using only APIs from core in your allocator
+        // and implementing your own system abstractions. Still, if you feel that
+        // a particular API should be entirely allocation-free, feel free to open
+        // an issue on the Rust repository, we'll see what we can do.
+        rtabort!(
+            "\n
+            Attempted to access thread-local data while allocating said data.\n
+            Do not access functions that allocate in the global allocator!\n
+            This is a bug in the global allocator.\n
+        "
+        )
+    } else {
+        debug_assert_eq!(current, DESTROYED);
+        panic!(
+            "use of std::thread::current() is not possible after the thread's
+         local data has been destroyed"
+        )
+    }
+}
+
+/// This should be run in [`crate::rt::thread_cleanup`] to reset the thread
+/// handle.
+pub(crate) fn drop_current() {
+    let current = CURRENT.get();
+    if current > DESTROYED {
+        unsafe {
+            CURRENT.set(DESTROYED);
+            drop(Thread::from_raw(current));
+        }
+    }
+}
diff --git a/library/std/src/thread/local/tests.rs b/library/std/src/thread/local/tests.rs
index 6abb9b85a2e..9d4f52a0921 100644
--- a/library/std/src/thread/local/tests.rs
+++ b/library/std/src/thread/local/tests.rs
@@ -1,7 +1,7 @@
 use crate::cell::{Cell, UnsafeCell};
 use crate::sync::atomic::{AtomicU8, Ordering};
 use crate::sync::{Arc, Condvar, Mutex};
-use crate::thread::{self, LocalKey};
+use crate::thread::{self, Builder, LocalKey};
 use crate::thread_local;
 
 #[derive(Clone, Default)]
@@ -343,3 +343,34 @@ fn join_orders_after_tls_destructors() {
         jh2.join().unwrap();
     }
 }
+
+// Test that thread::current is still available in TLS destructors.
+#[test]
+fn thread_current_in_dtor() {
+    // Go through one round of TLS destruction first.
+    struct Defer;
+    impl Drop for Defer {
+        fn drop(&mut self) {
+            RETRIEVE.with(|_| {});
+        }
+    }
+
+    struct RetrieveName;
+    impl Drop for RetrieveName {
+        fn drop(&mut self) {
+            *NAME.lock().unwrap() = Some(thread::current().name().unwrap().to_owned());
+        }
+    }
+
+    static NAME: Mutex<Option<String>> = Mutex::new(None);
+
+    thread_local! {
+        static DEFER: Defer = const { Defer };
+        static RETRIEVE: RetrieveName = const { RetrieveName };
+    }
+
+    Builder::new().name("test".to_owned()).spawn(|| DEFER.with(|_| {})).unwrap().join().unwrap();
+    let name = NAME.lock().unwrap();
+    let name = name.as_ref().unwrap();
+    assert_eq!(name, "test");
+}
diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs
index 22d65583365..d1d4eabb9bd 100644
--- a/library/std/src/thread/mod.rs
+++ b/library/std/src/thread/mod.rs
@@ -141,7 +141,7 @@
 //! [`Result`]: crate::result::Result
 //! [`Ok`]: crate::result::Result::Ok
 //! [`Err`]: crate::result::Result::Err
-//! [`thread::current`]: current
+//! [`thread::current`]: current::current
 //! [`thread::Result`]: Result
 //! [`unpark`]: Thread::unpark
 //! [`thread::park_timeout`]: park_timeout
@@ -159,7 +159,7 @@
 mod tests;
 
 use crate::any::Any;
-use crate::cell::{Cell, OnceCell, UnsafeCell};
+use crate::cell::UnsafeCell;
 use crate::ffi::CStr;
 use crate::marker::PhantomData;
 use crate::mem::{self, ManuallyDrop, forget};
@@ -179,6 +179,12 @@ mod scoped;
 #[stable(feature = "scoped_threads", since = "1.63.0")]
 pub use scoped::{Scope, ScopedJoinHandle, scope};
 
+mod current;
+
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use current::current;
+pub(crate) use current::{current_id, drop_current, set_current, try_current};
+
 ////////////////////////////////////////////////////////////////////////////////
 // Thread-local storage
 ////////////////////////////////////////////////////////////////////////////////
@@ -471,7 +477,11 @@ impl Builder {
             amt
         });
 
-        let my_thread = name.map_or_else(Thread::new_unnamed, Thread::new);
+        let id = ThreadId::new();
+        let my_thread = match name {
+            Some(name) => Thread::new(id, name.into()),
+            None => Thread::new_unnamed(id),
+        };
         let their_thread = my_thread.clone();
 
         let my_packet: Arc<Packet<'scope, T>> = Arc::new(Packet {
@@ -509,6 +519,9 @@ impl Builder {
 
         let f = MaybeDangling::new(f);
         let main = move || {
+            // Immediately store the thread handle to avoid setting it or its ID
+            // twice, which would cause an abort.
+            set_current(their_thread.clone());
             if let Some(name) = their_thread.cname() {
                 imp::Thread::set_name(name);
             }
@@ -516,7 +529,6 @@ impl Builder {
             crate::io::set_output_capture(output_capture);
 
             let f = f.into_inner();
-            set_current(their_thread);
             let try_result = panic::catch_unwind(panic::AssertUnwindSafe(|| {
                 crate::sys::backtrace::__rust_begin_short_backtrace(f)
             }));
@@ -690,84 +702,6 @@ where
     Builder::new().spawn(f).expect("failed to spawn thread")
 }
 
-thread_local! {
-    // Invariant: `CURRENT` and `CURRENT_ID` will always be initialized together.
-    // If `CURRENT` is initialized, then `CURRENT_ID` will hold the same value
-    // as `CURRENT.id()`.
-    static CURRENT: OnceCell<Thread> = const { OnceCell::new() };
-    static CURRENT_ID: Cell<Option<ThreadId>> = const { Cell::new(None) };
-}
-
-/// Sets the thread handle for the current thread.
-///
-/// Aborts if the handle has been set already to reduce code size.
-pub(crate) fn set_current(thread: Thread) {
-    let tid = thread.id();
-    // Using `unwrap` here can add ~3kB to the binary size. We have complete
-    // control over where this is called, so just abort if there is a bug.
-    CURRENT.with(|current| match current.set(thread) {
-        Ok(()) => CURRENT_ID.set(Some(tid)),
-        Err(_) => rtabort!("thread::set_current should only be called once per thread"),
-    });
-}
-
-/// Gets a handle to the thread that invokes it.
-///
-/// In contrast to the public `current` function, this will not panic if called
-/// from inside a TLS destructor.
-pub(crate) fn try_current() -> Option<Thread> {
-    CURRENT
-        .try_with(|current| {
-            current
-                .get_or_init(|| {
-                    let thread = Thread::new_unnamed();
-                    CURRENT_ID.set(Some(thread.id()));
-                    thread
-                })
-                .clone()
-        })
-        .ok()
-}
-
-/// Gets the id of the thread that invokes it.
-#[inline]
-pub(crate) fn current_id() -> ThreadId {
-    CURRENT_ID.get().unwrap_or_else(|| {
-        // If `CURRENT_ID` isn't initialized yet, then `CURRENT` must also not be initialized.
-        // `current()` will initialize both `CURRENT` and `CURRENT_ID` so subsequent calls to
-        // `current_id()` will succeed immediately.
-        current().id()
-    })
-}
-
-/// Gets a handle to the thread that invokes it.
-///
-/// # Examples
-///
-/// Getting a handle to the current thread with `thread::current()`:
-///
-/// ```
-/// use std::thread;
-///
-/// let handler = thread::Builder::new()
-///     .name("named thread".into())
-///     .spawn(|| {
-///         let handle = thread::current();
-///         assert_eq!(handle.name(), Some("named thread"));
-///     })
-///     .unwrap();
-///
-/// handler.join().unwrap();
-/// ```
-#[must_use]
-#[stable(feature = "rust1", since = "1.0.0")]
-pub fn current() -> Thread {
-    try_current().expect(
-        "use of std::thread::current() is not possible \
-         after the thread's local data has been destroyed",
-    )
-}
-
 /// Cooperatively gives up a timeslice to the OS scheduler.
 ///
 /// This calls the underlying OS scheduler's yield primitive, signaling
@@ -1225,8 +1159,11 @@ pub fn park_timeout(dur: Duration) {
 pub struct ThreadId(NonZero<u64>);
 
 impl ThreadId {
+    // DO NOT rely on this value.
+    const MAIN_THREAD: ThreadId = ThreadId(unsafe { NonZero::new_unchecked(1) });
+
     // Generate a new unique thread ID.
-    fn new() -> ThreadId {
+    pub(crate) fn new() -> ThreadId {
         #[cold]
         fn exhausted() -> ! {
             panic!("failed to generate unique thread ID: bitspace exhausted")
@@ -1236,7 +1173,7 @@ impl ThreadId {
             if #[cfg(target_has_atomic = "64")] {
                 use crate::sync::atomic::AtomicU64;
 
-                static COUNTER: AtomicU64 = AtomicU64::new(0);
+                static COUNTER: AtomicU64 = AtomicU64::new(1);
 
                 let mut last = COUNTER.load(Ordering::Relaxed);
                 loop {
@@ -1252,7 +1189,7 @@ impl ThreadId {
             } else {
                 use crate::sync::{Mutex, PoisonError};
 
-                static COUNTER: Mutex<u64> = Mutex::new(0);
+                static COUNTER: Mutex<u64> = Mutex::new(1);
 
                 let mut counter = COUNTER.lock().unwrap_or_else(PoisonError::into_inner);
                 let Some(id) = counter.checked_add(1) else {
@@ -1269,6 +1206,11 @@ impl ThreadId {
         }
     }
 
+    #[cfg(not(target_thread_local))]
+    fn from_u64(v: u64) -> Option<ThreadId> {
+        NonZero::new(v).map(ThreadId)
+    }
+
     /// This returns a numeric identifier for the thread identified by this
     /// `ThreadId`.
     ///
@@ -1369,27 +1311,27 @@ impl Inner {
 /// should instead use a function like `spawn` to create new threads, see the
 /// docs of [`Builder`] and [`spawn`] for more details.
 ///
-/// [`thread::current`]: current
+/// [`thread::current`]: current::current
 pub struct Thread {
     inner: Pin<Arc<Inner>>,
 }
 
 impl Thread {
     /// Used only internally to construct a thread object without spawning.
-    pub(crate) fn new(name: String) -> Thread {
-        Self::new_inner(ThreadName::Other(name.into()))
+    pub(crate) fn new(id: ThreadId, name: String) -> Thread {
+        Self::new_inner(id, ThreadName::Other(name.into()))
     }
 
-    pub(crate) fn new_unnamed() -> Thread {
-        Self::new_inner(ThreadName::Unnamed)
+    pub(crate) fn new_unnamed(id: ThreadId) -> Thread {
+        Self::new_inner(id, ThreadName::Unnamed)
     }
 
     // Used in runtime to construct main thread
     pub(crate) fn new_main() -> Thread {
-        Self::new_inner(ThreadName::Main)
+        Self::new_inner(ThreadId::MAIN_THREAD, ThreadName::Main)
     }
 
-    fn new_inner(name: ThreadName) -> Thread {
+    fn new_inner(id: ThreadId, name: ThreadName) -> Thread {
         // We have to use `unsafe` here to construct the `Parker` in-place,
         // which is required for the UNIX implementation.
         //
@@ -1399,7 +1341,7 @@ impl Thread {
             let mut arc = Arc::<Inner>::new_uninit();
             let ptr = Arc::get_mut_unchecked(&mut arc).as_mut_ptr();
             (&raw mut (*ptr).name).write(name);
-            (&raw mut (*ptr).id).write(ThreadId::new());
+            (&raw mut (*ptr).id).write(id);
             Parker::new_in_place(&raw mut (*ptr).parker);
             Pin::new_unchecked(arc.assume_init())
         };