diff options
| -rw-r--r-- | library/std/src/sys/common/thread_local.rs | 315 | ||||
| -rw-r--r-- | library/std/src/sys/common/thread_local/fast_local.rs | 276 | ||||
| -rw-r--r-- | library/std/src/sys/common/thread_local/mod.rs | 109 | ||||
| -rw-r--r-- | library/std/src/sys/common/thread_local/os_local.rs | 217 | ||||
| -rw-r--r-- | library/std/src/sys/common/thread_local/static_local.rs | 115 | ||||
| -rw-r--r-- | library/std/src/thread/local.rs | 373 | ||||
| -rw-r--r-- | library/std/src/thread/mod.rs | 37 | ||||
| -rw-r--r-- | src/tools/tidy/src/pal.rs | 2 | ||||
| -rw-r--r-- | tests/ui/threads-sendsync/issue-43733-2.rs | 2 | ||||
| -rw-r--r-- | tests/ui/threads-sendsync/issue-43733.rs | 8 |
10 files changed, 723 insertions, 731 deletions
diff --git a/library/std/src/sys/common/thread_local.rs b/library/std/src/sys/common/thread_local.rs deleted file mode 100644 index cddf0989348..00000000000 --- a/library/std/src/sys/common/thread_local.rs +++ /dev/null @@ -1,315 +0,0 @@ -#[doc(hidden)] -#[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] -#[macro_export] -#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] -#[allow_internal_unsafe] -#[cfg(all( - not(target_thread_local), - not(all(target_family = "wasm", not(target_feature = "atomics"))) -))] -macro_rules! __thread_local_inner { - // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[cfg_attr(not(windows), inline)] // see comments below - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // On platforms without `#[thread_local]` we fall back to the - // same implementation as below for os thread locals. - #[inline] - const fn __init() -> $t { INIT_EXPR } - static __KEY: $crate::thread::__OsLocalKeyInner<$t> = - $crate::thread::__OsLocalKeyInner::new(); - #[allow(unused_unsafe)] - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = _init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing initial value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}; - - // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - - // When reading this function you might ask "why is this inlined - // everywhere other than Windows?", and that's a very reasonable - // question to ask. The short story is that it segfaults rustc if - // this function is inlined. The longer story is that Windows looks - // to not support `extern` references to thread locals across DLL - // boundaries. This appears to at least not be supported in the ABI - // that LLVM implements. - // - // Because of this we never inline on Windows, but we do inline on - // other platforms (where external references to thread locals - // across DLLs are supported). A better fix for this would be to - // inline this function on Windows, but only for "statically linked" - // components. For example if two separately compiled rlibs end up - // getting linked into a DLL then it's fine to inline this function - // across that boundary. It's only not fine to inline this function - // across a DLL boundary. Unfortunately rustc doesn't currently - // have this sort of logic available in an attribute, and it's not - // clear that rustc is even equipped to answer this (it's more of a - // Cargo question kinda). This means that, unfortunately, Windows - // gets the pessimistic path for now where it's never inlined. - // - // The issue of "should enable on Windows sometimes" is #84933 - #[cfg_attr(not(windows), inline)] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::__OsLocalKeyInner<$t> = - $crate::thread::__OsLocalKeyInner::new(); - - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - } - }; - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { - $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::__thread_local_inner!(@key $t, $($init)*); - } -} - -#[doc(hidden)] -#[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] -#[macro_export] -#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] -#[allow_internal_unsafe] -#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))] -macro_rules! __thread_local_inner { - // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[cfg_attr(not(windows), inline)] // see comments below - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - // If the platform has support for `#[thread_local]`, use it. - #[thread_local] - static mut VAL: $t = INIT_EXPR; - - // If a dtor isn't needed we can do something "very raw" and - // just get going. - if !$crate::mem::needs_drop::<$t>() { - unsafe { - return $crate::option::Option::Some(&VAL) - } - } - - // 0 == dtor not registered - // 1 == dtor registered, dtor not run - // 2 == dtor registered and is running or has run - #[thread_local] - static mut STATE: $crate::primitive::u8 = 0; - - unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { - let ptr = ptr as *mut $t; - - unsafe { - $crate::debug_assert_eq!(STATE, 1); - STATE = 2; - $crate::ptr::drop_in_place(ptr); - } - } - - unsafe { - match STATE { - // 0 == we haven't registered a destructor, so do - // so now. - 0 => { - $crate::thread::__FastLocalKeyInner::<$t>::register_dtor( - $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8, - destroy, - ); - STATE = 1; - $crate::option::Option::Some(&VAL) - } - // 1 == the destructor is registered and the value - // is valid, so return the pointer. - 1 => $crate::option::Option::Some(&VAL), - // otherwise the destructor has already run, so we - // can't give access. - _ => $crate::option::Option::None, - } - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}; - - // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - - // When reading this function you might ask "why is this inlined - // everywhere other than Windows?", and that's a very reasonable - // question to ask. The short story is that it segfaults rustc if - // this function is inlined. The longer story is that Windows looks - // to not support `extern` references to thread locals across DLL - // boundaries. This appears to at least not be supported in the ABI - // that LLVM implements. - // - // Because of this we never inline on Windows, but we do inline on - // other platforms (where external references to thread locals - // across DLLs are supported). A better fix for this would be to - // inline this function on Windows, but only for "statically linked" - // components. For example if two separately compiled rlibs end up - // getting linked into a DLL then it's fine to inline this function - // across that boundary. It's only not fine to inline this function - // across a DLL boundary. Unfortunately rustc doesn't currently - // have this sort of logic available in an attribute, and it's not - // clear that rustc is even equipped to answer this (it's more of a - // Cargo question kinda). This means that, unfortunately, Windows - // gets the pessimistic path for now where it's never inlined. - // - // The issue of "should enable on Windows sometimes" is #84933 - #[cfg_attr(not(windows), inline)] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - #[thread_local] - static __KEY: $crate::thread::__FastLocalKeyInner<$t> = - $crate::thread::__FastLocalKeyInner::new(); - - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - } - }; - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { - $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::__thread_local_inner!(@key $t, $($init)*); - } -} - -#[doc(hidden)] -#[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] -#[macro_export] -#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] -#[allow_internal_unsafe] -#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] -macro_rules! __thread_local_inner { - // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[inline] // see comments below - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // wasm without atomics maps directly to `static mut`, and dtors - // aren't implemented because thread dtors aren't really a thing - // on wasm right now - // - // FIXME(#84224) this should come after the `target_thread_local` - // block. - static mut VAL: $t = INIT_EXPR; - unsafe { $crate::option::Option::Some(&VAL) } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}; - - // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - #[inline] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::__StaticLocalKeyInner<$t> = - $crate::thread::__StaticLocalKeyInner::new(); - - // FIXME: remove the #[allow(...)] marker when macros don't - // raise warning for missing/extraneous unsafe blocks anymore. - // See https://github.com/rust-lang/rust/issues/74838. - #[allow(unused_unsafe)] - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - } - }; - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { - $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::__thread_local_inner!(@key $t, $($init)*); - } -} diff --git a/library/std/src/sys/common/thread_local/fast_local.rs b/library/std/src/sys/common/thread_local/fast_local.rs new file mode 100644 index 00000000000..2addcc4a759 --- /dev/null +++ b/library/std/src/sys/common/thread_local/fast_local.rs @@ -0,0 +1,276 @@ +#[doc(hidden)] +#[macro_export] +#[allow_internal_unstable( + thread_local_internals, + cfg_target_thread_local, + thread_local, + libstd_thread_internals +)] +#[allow_internal_unsafe] +macro_rules! __thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + #[cfg_attr(not(windows), inline)] // see comments below + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + const INIT_EXPR: $t = $init; + // If the platform has support for `#[thread_local]`, use it. + #[thread_local] + static mut VAL: $t = INIT_EXPR; + + // If a dtor isn't needed we can do something "very raw" and + // just get going. + if !$crate::mem::needs_drop::<$t>() { + unsafe { + return $crate::option::Option::Some(&VAL) + } + } + + // 0 == dtor not registered + // 1 == dtor registered, dtor not run + // 2 == dtor registered and is running or has run + #[thread_local] + static mut STATE: $crate::primitive::u8 = 0; + + unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { + let ptr = ptr as *mut $t; + + unsafe { + $crate::debug_assert_eq!(STATE, 1); + STATE = 2; + $crate::ptr::drop_in_place(ptr); + } + } + + unsafe { + match STATE { + // 0 == we haven't registered a destructor, so do + // so now. + 0 => { + $crate::thread::__LocalKeyInner::<$t>::register_dtor( + $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8, + destroy, + ); + STATE = 1; + $crate::option::Option::Some(&VAL) + } + // 1 == the destructor is registered and the value + // is valid, so return the pointer. + 1 => $crate::option::Option::Some(&VAL), + // otherwise the destructor has already run, so we + // can't give access. + _ => $crate::option::Option::None, + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}; + + // used to generate the `LocalKey` value for `thread_local!` + (@key $t:ty, $init:expr) => { + { + #[inline] + fn __init() -> $t { $init } + + // When reading this function you might ask "why is this inlined + // everywhere other than Windows?", and that's a very reasonable + // question to ask. The short story is that it segfaults rustc if + // this function is inlined. The longer story is that Windows looks + // to not support `extern` references to thread locals across DLL + // boundaries. This appears to at least not be supported in the ABI + // that LLVM implements. + // + // Because of this we never inline on Windows, but we do inline on + // other platforms (where external references to thread locals + // across DLLs are supported). A better fix for this would be to + // inline this function on Windows, but only for "statically linked" + // components. For example if two separately compiled rlibs end up + // getting linked into a DLL then it's fine to inline this function + // across that boundary. It's only not fine to inline this function + // across a DLL boundary. Unfortunately rustc doesn't currently + // have this sort of logic available in an attribute, and it's not + // clear that rustc is even equipped to answer this (it's more of a + // Cargo question kinda). This means that, unfortunately, Windows + // gets the pessimistic path for now where it's never inlined. + // + // The issue of "should enable on Windows sometimes" is #84933 + #[cfg_attr(not(windows), inline)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + #[thread_local] + static __KEY: $crate::thread::__LocalKeyInner<$t> = + $crate::thread::__LocalKeyInner::<$t>::new(); + + // FIXME: remove the #[allow(...)] marker when macros don't + // raise warning for missing/extraneous unsafe blocks anymore. + // See https://github.com/rust-lang/rust/issues/74838. + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing default value"); + } + } + __init() + }) + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + } + }; + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { + $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = + $crate::__thread_local_inner!(@key $t, $($init)*); + } +} + +#[doc(hidden)] +pub mod fast { + use super::super::lazy::LazyKeyInner; + use crate::cell::Cell; + use crate::sys::thread_local_dtor::register_dtor; + use crate::{fmt, mem, panic}; + + #[derive(Copy, Clone)] + enum DtorState { + Unregistered, + Registered, + RunningOrHasRun, + } + + // This data structure has been carefully constructed so that the fast path + // only contains one branch on x86. That optimization is necessary to avoid + // duplicated tls lookups on OSX. + // + // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 + pub struct Key<T> { + // If `LazyKeyInner::get` returns `None`, that indicates either: + // * The value has never been initialized + // * The value is being recursively initialized + // * The value has already been destroyed or is being destroyed + // To determine which kind of `None`, check `dtor_state`. + // + // This is very optimizer friendly for the fast path - initialized but + // not yet dropped. + inner: LazyKeyInner<T>, + + // Metadata to keep track of the state of the destructor. Remember that + // this variable is thread-local, not global. + dtor_state: Cell<DtorState>, + } + + impl<T> fmt::Debug for Key<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Key").finish_non_exhaustive() + } + } + + impl<T> Key<T> { + pub const fn new() -> Key<T> { + Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } + } + + // note that this is just a publicly-callable function only for the + // const-initialized form of thread locals, basically a way to call the + // free `register_dtor` function defined elsewhere in std. + pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { + unsafe { + register_dtor(a, dtor); + } + } + + pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { + // SAFETY: See the definitions of `LazyKeyInner::get` and + // `try_initialize` for more information. + // + // The caller must ensure no mutable references are ever active to + // the inner cell or the inner T when this is called. + // The `try_initialize` is dependant on the passed `init` function + // for this. + unsafe { + match self.inner.get() { + Some(val) => Some(val), + None => self.try_initialize(init), + } + } + } + + // `try_initialize` is only called once per fast thread local variable, + // except in corner cases where thread_local dtors reference other + // thread_local's, or it is being recursively initialized. + // + // Macos: Inlining this function can cause two `tlv_get_addr` calls to + // be performed for every call to `Key::get`. + // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 + #[inline(never)] + unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { + // SAFETY: See comment above (this function doc). + if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { + // SAFETY: See comment above (this function doc). + Some(unsafe { self.inner.initialize(init) }) + } else { + None + } + } + + // `try_register_dtor` is only called once per fast thread local + // variable, except in corner cases where thread_local dtors reference + // other thread_local's, or it is being recursively initialized. + unsafe fn try_register_dtor(&self) -> bool { + match self.dtor_state.get() { + DtorState::Unregistered => { + // SAFETY: dtor registration happens before initialization. + // Passing `self` as a pointer while using `destroy_value<T>` + // is safe because the function will build a pointer to a + // Key<T>, which is the type of self and so find the correct + // size. + unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) }; + self.dtor_state.set(DtorState::Registered); + true + } + DtorState::Registered => { + // recursively initialized + true + } + DtorState::RunningOrHasRun => false, + } + } + } + + unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { + let ptr = ptr as *mut Key<T>; + + // SAFETY: + // + // The pointer `ptr` has been built just above and comes from + // `try_register_dtor` where it is originally a Key<T> coming from `self`, + // making it non-NUL and of the correct type. + // + // Right before we run the user destructor be sure to set the + // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This + // causes future calls to `get` to run `try_initialize_drop` again, + // which will now fail, and return `None`. + // + // Wrap the call in a catch to ensure unwinding is caught in the event + // a panic takes place in a destructor. + if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { + let value = (*ptr).inner.take(); + (*ptr).dtor_state.set(DtorState::RunningOrHasRun); + drop(value); + })) { + rtabort!("thread local panicked on drop"); + } + } +} diff --git a/library/std/src/sys/common/thread_local/mod.rs b/library/std/src/sys/common/thread_local/mod.rs new file mode 100644 index 00000000000..1fee84a0434 --- /dev/null +++ b/library/std/src/sys/common/thread_local/mod.rs @@ -0,0 +1,109 @@ +//! The following module declarations are outside cfg_if because the internal +//! `__thread_local_internal` macro does not seem to be exported properly when using cfg_if +#![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] + +#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))] +mod fast_local; +#[cfg(all( + not(target_thread_local), + not(all(target_family = "wasm", not(target_feature = "atomics"))) +))] +mod os_local; +#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] +mod static_local; + +#[cfg(not(test))] +cfg_if::cfg_if! { + if #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] { + #[doc(hidden)] + pub use static_local::statik::Key; + } else if #[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))] { + #[doc(hidden)] + pub use fast_local::fast::Key; + } else if #[cfg(all(not(target_thread_local), not(all(target_family = "wasm", not(target_feature = "atomics")))))] { + #[doc(hidden)] + pub use os_local::os::Key; + } +} + +#[doc(hidden)] +#[cfg(test)] +pub use realstd::thread::__LocalKeyInner as Key; + +mod lazy { + use crate::cell::UnsafeCell; + use crate::hint; + use crate::mem; + + pub struct LazyKeyInner<T> { + inner: UnsafeCell<Option<T>>, + } + + impl<T> LazyKeyInner<T> { + pub const fn new() -> LazyKeyInner<T> { + LazyKeyInner { inner: UnsafeCell::new(None) } + } + + pub unsafe fn get(&self) -> Option<&'static T> { + // SAFETY: The caller must ensure no reference is ever handed out to + // the inner cell nor mutable reference to the Option<T> inside said + // cell. This make it safe to hand a reference, though the lifetime + // of 'static is itself unsafe, making the get method unsafe. + unsafe { (*self.inner.get()).as_ref() } + } + + /// The caller must ensure that no reference is active: this method + /// needs unique access. + pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T { + // Execute the initialization up front, *then* move it into our slot, + // just in case initialization fails. + let value = init(); + let ptr = self.inner.get(); + + // SAFETY: + // + // note that this can in theory just be `*ptr = Some(value)`, but due to + // the compiler will currently codegen that pattern with something like: + // + // ptr::drop_in_place(ptr) + // ptr::write(ptr, Some(value)) + // + // Due to this pattern it's possible for the destructor of the value in + // `ptr` (e.g., if this is being recursively initialized) to re-access + // TLS, in which case there will be a `&` and `&mut` pointer to the same + // value (an aliasing violation). To avoid setting the "I'm running a + // destructor" flag we just use `mem::replace` which should sequence the + // operations a little differently and make this safe to call. + // + // The precondition also ensures that we are the only one accessing + // `self` at the moment so replacing is fine. + unsafe { + let _ = mem::replace(&mut *ptr, Some(value)); + } + + // SAFETY: With the call to `mem::replace` it is guaranteed there is + // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` + // will never be reached. + unsafe { + // After storing `Some` we want to get a reference to the contents of + // what we just stored. While we could use `unwrap` here and it should + // always work it empirically doesn't seem to always get optimized away, + // which means that using something like `try_with` can pull in + // panicking code and cause a large size bloat. + match *ptr { + Some(ref x) => x, + None => hint::unreachable_unchecked(), + } + } + } + + /// The other methods hand out references while taking &self. + /// As such, callers of this method must ensure no `&` and `&mut` are + /// available and used at the same time. + #[allow(unused)] + pub unsafe fn take(&mut self) -> Option<T> { + // SAFETY: See doc comment for this method. + unsafe { (*self.inner.get()).take() } + } + } +} diff --git a/library/std/src/sys/common/thread_local/os_local.rs b/library/std/src/sys/common/thread_local/os_local.rs new file mode 100644 index 00000000000..6f6560c4aa9 --- /dev/null +++ b/library/std/src/sys/common/thread_local/os_local.rs @@ -0,0 +1,217 @@ +#[doc(hidden)] +#[macro_export] +#[allow_internal_unstable( + thread_local_internals, + cfg_target_thread_local, + thread_local, + libstd_thread_internals +)] +#[allow_internal_unsafe] +macro_rules! __thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + #[cfg_attr(not(windows), inline)] // see comments below + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + const INIT_EXPR: $t = $init; + + // On platforms without `#[thread_local]` we fall back to the + // same implementation as below for os thread locals. + #[inline] + const fn __init() -> $t { INIT_EXPR } + static __KEY: $crate::thread::__LocalKeyInner<$t> = + $crate::thread::__LocalKeyInner::new(); + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = _init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing initial value"); + } + } + __init() + }) + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}; + + // used to generate the `LocalKey` value for `thread_local!` + (@key $t:ty, $init:expr) => { + { + #[inline] + fn __init() -> $t { $init } + + // When reading this function you might ask "why is this inlined + // everywhere other than Windows?", and that's a very reasonable + // question to ask. The short story is that it segfaults rustc if + // this function is inlined. The longer story is that Windows looks + // to not support `extern` references to thread locals across DLL + // boundaries. This appears to at least not be supported in the ABI + // that LLVM implements. + // + // Because of this we never inline on Windows, but we do inline on + // other platforms (where external references to thread locals + // across DLLs are supported). A better fix for this would be to + // inline this function on Windows, but only for "statically linked" + // components. For example if two separately compiled rlibs end up + // getting linked into a DLL then it's fine to inline this function + // across that boundary. It's only not fine to inline this function + // across a DLL boundary. Unfortunately rustc doesn't currently + // have this sort of logic available in an attribute, and it's not + // clear that rustc is even equipped to answer this (it's more of a + // Cargo question kinda). This means that, unfortunately, Windows + // gets the pessimistic path for now where it's never inlined. + // + // The issue of "should enable on Windows sometimes" is #84933 + #[cfg_attr(not(windows), inline)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + static __KEY: $crate::thread::__LocalKeyInner<$t> = + $crate::thread::__LocalKeyInner::new(); + + // FIXME: remove the #[allow(...)] marker when macros don't + // raise warning for missing/extraneous unsafe blocks anymore. + // See https://github.com/rust-lang/rust/issues/74838. + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing default value"); + } + } + __init() + }) + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + } + }; + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { + $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = + $crate::__thread_local_inner!(@key $t, $($init)*); + } +} + +#[doc(hidden)] +pub mod os { + use super::super::lazy::LazyKeyInner; + use crate::cell::Cell; + use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; + use crate::{fmt, marker, panic, ptr}; + + /// Use a regular global static to store this key; the state provided will then be + /// thread-local. + pub struct Key<T> { + // OS-TLS key that we'll use to key off. + os: OsStaticKey, + marker: marker::PhantomData<Cell<T>>, + } + + impl<T> fmt::Debug for Key<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Key").finish_non_exhaustive() + } + } + + unsafe impl<T> Sync for Key<T> {} + + struct Value<T: 'static> { + inner: LazyKeyInner<T>, + key: &'static Key<T>, + } + + impl<T: 'static> Key<T> { + #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] + pub const fn new() -> Key<T> { + Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData } + } + + /// It is a requirement for the caller to ensure that no mutable + /// reference is active when this method is called. + pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: See the documentation for this method. + let ptr = unsafe { self.os.get() as *mut Value<T> }; + if ptr.addr() > 1 { + // SAFETY: the check ensured the pointer is safe (its destructor + // is not running) + it is coming from a trusted source (self). + if let Some(ref value) = unsafe { (*ptr).inner.get() } { + return Some(value); + } + } + // SAFETY: At this point we are sure we have no value and so + // initializing (or trying to) is safe. + unsafe { self.try_initialize(init) } + } + + // `try_initialize` is only called once per os thread local variable, + // except in corner cases where thread_local dtors reference other + // thread_local's, or it is being recursively initialized. + unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: No mutable references are ever handed out meaning getting + // the value is ok. + let ptr = unsafe { self.os.get() as *mut Value<T> }; + if ptr.addr() == 1 { + // destructor is running + return None; + } + + let ptr = if ptr.is_null() { + // If the lookup returned null, we haven't initialized our own + // local copy, so do that now. + let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self })); + // SAFETY: At this point we are sure there is no value inside + // ptr so setting it will not affect anyone else. + unsafe { + self.os.set(ptr as *mut u8); + } + ptr + } else { + // recursive initialization + ptr + }; + + // SAFETY: ptr has been ensured as non-NUL just above an so can be + // dereferenced safely. + unsafe { Some((*ptr).inner.initialize(init)) } + } + } + + unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { + // SAFETY: + // + // The OS TLS ensures that this key contains a null value when this + // destructor starts to run. We set it back to a sentinel value of 1 to + // ensure that any future calls to `get` for this thread will return + // `None`. + // + // Note that to prevent an infinite loop we reset it back to null right + // before we return from the destructor ourselves. + // + // Wrap the call in a catch to ensure unwinding is caught in the event + // a panic takes place in a destructor. + if let Err(_) = panic::catch_unwind(|| unsafe { + let ptr = Box::from_raw(ptr as *mut Value<T>); + let key = ptr.key; + key.os.set(ptr::invalid_mut(1)); + drop(ptr); + key.os.set(ptr::null_mut()); + }) { + rtabort!("thread local panicked on drop"); + } + } +} diff --git a/library/std/src/sys/common/thread_local/static_local.rs b/library/std/src/sys/common/thread_local/static_local.rs new file mode 100644 index 00000000000..ec4f2a12b7e --- /dev/null +++ b/library/std/src/sys/common/thread_local/static_local.rs @@ -0,0 +1,115 @@ +#[doc(hidden)] +#[macro_export] +#[allow_internal_unstable( + thread_local_internals, + cfg_target_thread_local, + thread_local, + libstd_thread_internals +)] +#[allow_internal_unsafe] +macro_rules! __thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + #[inline] // see comments below + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + const INIT_EXPR: $t = $init; + + // wasm without atomics maps directly to `static mut`, and dtors + // aren't implemented because thread dtors aren't really a thing + // on wasm right now + // + // FIXME(#84224) this should come after the `target_thread_local` + // block. + static mut VAL: $t = INIT_EXPR; + unsafe { $crate::option::Option::Some(&VAL) } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}; + + // used to generate the `LocalKey` value for `thread_local!` + (@key $t:ty, $init:expr) => { + { + #[inline] + fn __init() -> $t { $init } + #[inline] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + static __KEY: $crate::thread::__LocalKeyInner<$t> = + $crate::thread::__LocalKeyInner::new(); + + // FIXME: remove the #[allow(...)] marker when macros don't + // raise warning for missing/extraneous unsafe blocks anymore. + // See https://github.com/rust-lang/rust/issues/74838. + #[allow(unused_unsafe)] + unsafe { + __KEY.get(move || { + if let $crate::option::Option::Some(init) = init { + if let $crate::option::Option::Some(value) = init.take() { + return value; + } else if $crate::cfg!(debug_assertions) { + $crate::unreachable!("missing default value"); + } + } + __init() + }) + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + } + }; + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { + $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = + $crate::__thread_local_inner!(@key $t, $($init)*); + } +} + +/// On some targets like wasm there's no threads, so no need to generate +/// thread locals and we can instead just use plain statics! +#[doc(hidden)] +pub mod statik { + use super::super::lazy::LazyKeyInner; + use crate::fmt; + + pub struct Key<T> { + inner: LazyKeyInner<T>, + } + + unsafe impl<T> Sync for Key<T> {} + + impl<T> fmt::Debug for Key<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Key").finish_non_exhaustive() + } + } + + impl<T> Key<T> { + pub const fn new() -> Key<T> { + Key { inner: LazyKeyInner::new() } + } + + pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { + // SAFETY: The caller must ensure no reference is ever handed out to + // the inner cell nor mutable reference to the Option<T> inside said + // cell. This make it safe to hand a reference, though the lifetime + // of 'static is itself unsafe, making the get method unsafe. + let value = unsafe { + match self.inner.get() { + Some(ref value) => value, + None => self.inner.initialize(init), + } + }; + + Some(value) + } + } +} diff --git a/library/std/src/thread/local.rs b/library/std/src/thread/local.rs index c82b5c2284f..7fdf03acc14 100644 --- a/library/std/src/thread/local.rs +++ b/library/std/src/thread/local.rs @@ -585,376 +585,3 @@ impl<T: 'static> LocalKey<RefCell<T>> { self.with(|cell| cell.replace(value)) } } - -mod lazy { - use crate::cell::UnsafeCell; - use crate::hint; - use crate::mem; - - pub struct LazyKeyInner<T> { - inner: UnsafeCell<Option<T>>, - } - - impl<T> LazyKeyInner<T> { - pub const fn new() -> LazyKeyInner<T> { - LazyKeyInner { inner: UnsafeCell::new(None) } - } - - pub unsafe fn get(&self) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - unsafe { (*self.inner.get()).as_ref() } - } - - /// The caller must ensure that no reference is active: this method - /// needs unique access. - pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T { - // Execute the initialization up front, *then* move it into our slot, - // just in case initialization fails. - let value = init(); - let ptr = self.inner.get(); - - // SAFETY: - // - // note that this can in theory just be `*ptr = Some(value)`, but due to - // the compiler will currently codegen that pattern with something like: - // - // ptr::drop_in_place(ptr) - // ptr::write(ptr, Some(value)) - // - // Due to this pattern it's possible for the destructor of the value in - // `ptr` (e.g., if this is being recursively initialized) to re-access - // TLS, in which case there will be a `&` and `&mut` pointer to the same - // value (an aliasing violation). To avoid setting the "I'm running a - // destructor" flag we just use `mem::replace` which should sequence the - // operations a little differently and make this safe to call. - // - // The precondition also ensures that we are the only one accessing - // `self` at the moment so replacing is fine. - unsafe { - let _ = mem::replace(&mut *ptr, Some(value)); - } - - // SAFETY: With the call to `mem::replace` it is guaranteed there is - // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` - // will never be reached. - unsafe { - // After storing `Some` we want to get a reference to the contents of - // what we just stored. While we could use `unwrap` here and it should - // always work it empirically doesn't seem to always get optimized away, - // which means that using something like `try_with` can pull in - // panicking code and cause a large size bloat. - match *ptr { - Some(ref x) => x, - None => hint::unreachable_unchecked(), - } - } - } - - /// The other methods hand out references while taking &self. - /// As such, callers of this method must ensure no `&` and `&mut` are - /// available and used at the same time. - #[allow(unused)] - pub unsafe fn take(&mut self) -> Option<T> { - // SAFETY: See doc comment for this method. - unsafe { (*self.inner.get()).take() } - } - } -} - -/// On some targets like wasm there's no threads, so no need to generate -/// thread locals and we can instead just use plain statics! -#[doc(hidden)] -#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] -pub mod statik { - use super::lazy::LazyKeyInner; - use crate::fmt; - - pub struct Key<T> { - inner: LazyKeyInner<T>, - } - - unsafe impl<T> Sync for Key<T> {} - - impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } - } - - impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new() } - } - - pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - let value = unsafe { - match self.inner.get() { - Some(ref value) => value, - None => self.inner.initialize(init), - } - }; - - Some(value) - } - } -} - -#[doc(hidden)] -#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics"))),))] -pub mod fast { - use super::lazy::LazyKeyInner; - use crate::cell::Cell; - use crate::sys::thread_local_dtor::register_dtor; - use crate::{fmt, mem, panic}; - - #[derive(Copy, Clone)] - enum DtorState { - Unregistered, - Registered, - RunningOrHasRun, - } - - // This data structure has been carefully constructed so that the fast path - // only contains one branch on x86. That optimization is necessary to avoid - // duplicated tls lookups on OSX. - // - // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 - pub struct Key<T> { - // If `LazyKeyInner::get` returns `None`, that indicates either: - // * The value has never been initialized - // * The value is being recursively initialized - // * The value has already been destroyed or is being destroyed - // To determine which kind of `None`, check `dtor_state`. - // - // This is very optimizer friendly for the fast path - initialized but - // not yet dropped. - inner: LazyKeyInner<T>, - - // Metadata to keep track of the state of the destructor. Remember that - // this variable is thread-local, not global. - dtor_state: Cell<DtorState>, - } - - impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } - } - - impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } - } - - // note that this is just a publicly-callable function only for the - // const-initialized form of thread locals, basically a way to call the - // free `register_dtor` function defined elsewhere in std. - pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - unsafe { - register_dtor(a, dtor); - } - } - - pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See the definitions of `LazyKeyInner::get` and - // `try_initialize` for more information. - // - // The caller must ensure no mutable references are ever active to - // the inner cell or the inner T when this is called. - // The `try_initialize` is dependant on the passed `init` function - // for this. - unsafe { - match self.inner.get() { - Some(val) => Some(val), - None => self.try_initialize(init), - } - } - } - - // `try_initialize` is only called once per fast thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - // - // Macos: Inlining this function can cause two `tlv_get_addr` calls to - // be performed for every call to `Key::get`. - // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 - #[inline(never)] - unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See comment above (this function doc). - if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { - // SAFETY: See comment above (this function doc). - Some(unsafe { self.inner.initialize(init) }) - } else { - None - } - } - - // `try_register_dtor` is only called once per fast thread local - // variable, except in corner cases where thread_local dtors reference - // other thread_local's, or it is being recursively initialized. - unsafe fn try_register_dtor(&self) -> bool { - match self.dtor_state.get() { - DtorState::Unregistered => { - // SAFETY: dtor registration happens before initialization. - // Passing `self` as a pointer while using `destroy_value<T>` - // is safe because the function will build a pointer to a - // Key<T>, which is the type of self and so find the correct - // size. - unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) }; - self.dtor_state.set(DtorState::Registered); - true - } - DtorState::Registered => { - // recursively initialized - true - } - DtorState::RunningOrHasRun => false, - } - } - } - - unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { - let ptr = ptr as *mut Key<T>; - - // SAFETY: - // - // The pointer `ptr` has been built just above and comes from - // `try_register_dtor` where it is originally a Key<T> coming from `self`, - // making it non-NUL and of the correct type. - // - // Right before we run the user destructor be sure to set the - // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This - // causes future calls to `get` to run `try_initialize_drop` again, - // which will now fail, and return `None`. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { - let value = (*ptr).inner.take(); - (*ptr).dtor_state.set(DtorState::RunningOrHasRun); - drop(value); - })) { - rtabort!("thread local panicked on drop"); - } - } -} - -#[doc(hidden)] -#[cfg(all( - not(target_thread_local), - not(all(target_family = "wasm", not(target_feature = "atomics"))), -))] -pub mod os { - use super::lazy::LazyKeyInner; - use crate::cell::Cell; - use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; - use crate::{fmt, marker, panic, ptr}; - - /// Use a regular global static to store this key; the state provided will then be - /// thread-local. - pub struct Key<T> { - // OS-TLS key that we'll use to key off. - os: OsStaticKey, - marker: marker::PhantomData<Cell<T>>, - } - - impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } - } - - unsafe impl<T> Sync for Key<T> {} - - struct Value<T: 'static> { - inner: LazyKeyInner<T>, - key: &'static Key<T>, - } - - impl<T: 'static> Key<T> { - #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] - pub const fn new() -> Key<T> { - Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData } - } - - /// It is a requirement for the caller to ensure that no mutable - /// reference is active when this method is called. - pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: See the documentation for this method. - let ptr = unsafe { self.os.get() as *mut Value<T> }; - if ptr.addr() > 1 { - // SAFETY: the check ensured the pointer is safe (its destructor - // is not running) + it is coming from a trusted source (self). - if let Some(ref value) = unsafe { (*ptr).inner.get() } { - return Some(value); - } - } - // SAFETY: At this point we are sure we have no value and so - // initializing (or trying to) is safe. - unsafe { self.try_initialize(init) } - } - - // `try_initialize` is only called once per os thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: No mutable references are ever handed out meaning getting - // the value is ok. - let ptr = unsafe { self.os.get() as *mut Value<T> }; - if ptr.addr() == 1 { - // destructor is running - return None; - } - - let ptr = if ptr.is_null() { - // If the lookup returned null, we haven't initialized our own - // local copy, so do that now. - let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self })); - // SAFETY: At this point we are sure there is no value inside - // ptr so setting it will not affect anyone else. - unsafe { - self.os.set(ptr as *mut u8); - } - ptr - } else { - // recursive initialization - ptr - }; - - // SAFETY: ptr has been ensured as non-NUL just above an so can be - // dereferenced safely. - unsafe { Some((*ptr).inner.initialize(init)) } - } - } - - unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { - // SAFETY: - // - // The OS TLS ensures that this key contains a null value when this - // destructor starts to run. We set it back to a sentinel value of 1 to - // ensure that any future calls to `get` for this thread will return - // `None`. - // - // Note that to prevent an infinite loop we reset it back to null right - // before we return from the destructor ourselves. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(|| unsafe { - let ptr = Box::from_raw(ptr as *mut Value<T>); - let key = ptr.key; - key.os.set(ptr::invalid_mut(1)); - drop(ptr); - key.os.set(ptr::null_mut()); - }) { - rtabort!("thread local panicked on drop"); - } - } -} diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs index 489af776798..b9aaf5f6e15 100644 --- a/library/std/src/thread/mod.rs +++ b/library/std/src/thread/mod.rs @@ -203,44 +203,9 @@ pub use self::local::{AccessError, LocalKey}; // by the elf linker. "static" is for single-threaded platforms where a global // static is sufficient. -#[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(not(test))] -#[cfg(all( - target_thread_local, - not(all(target_family = "wasm", not(target_feature = "atomics"))), -))] -#[doc(hidden)] -pub use self::local::fast::Key as __FastLocalKeyInner; -// when building for tests, use real std's type -#[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(test)] -#[cfg(all( - target_thread_local, - not(all(target_family = "wasm", not(target_feature = "atomics"))), -))] -pub use realstd::thread::__FastLocalKeyInner; - -#[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(not(test))] -#[cfg(all( - not(target_thread_local), - not(all(target_family = "wasm", not(target_feature = "atomics"))), -))] #[doc(hidden)] -pub use self::local::os::Key as __OsLocalKeyInner; -// when building for tests, use real std's type -#[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(test)] -#[cfg(all( - not(target_thread_local), - not(all(target_family = "wasm", not(target_feature = "atomics"))), -))] -pub use realstd::thread::__OsLocalKeyInner; - #[unstable(feature = "libstd_thread_internals", issue = "none")] -#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] -#[doc(hidden)] -pub use self::local::statik::Key as __StaticLocalKeyInner; +pub use crate::sys::common::thread_local::Key as __LocalKeyInner; //////////////////////////////////////////////////////////////////////////////// // Builder diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs index 6d6d3c89a3c..868579b4b1a 100644 --- a/src/tools/tidy/src/pal.rs +++ b/src/tools/tidy/src/pal.rs @@ -62,8 +62,6 @@ const EXCEPTION_PATHS: &[&str] = &[ "library/std/src/panic.rs", // fuchsia-specific panic backtrace handling "library/std/src/personality.rs", "library/std/src/personality/", - "library/std/src/thread/mod.rs", - "library/std/src/thread/local.rs", ]; pub fn check(path: &Path, bad: &mut bool) { diff --git a/tests/ui/threads-sendsync/issue-43733-2.rs b/tests/ui/threads-sendsync/issue-43733-2.rs index 32baeec4359..8f7a9c08375 100644 --- a/tests/ui/threads-sendsync/issue-43733-2.rs +++ b/tests/ui/threads-sendsync/issue-43733-2.rs @@ -21,7 +21,7 @@ impl<T> Key<T> { } #[cfg(target_thread_local)] -use std::thread::__FastLocalKeyInner as Key; +use std::thread::__LocalKeyInner as Key; static __KEY: Key<()> = Key::new(); //~^ ERROR `UnsafeCell<Option<()>>` cannot be shared between threads diff --git a/tests/ui/threads-sendsync/issue-43733.rs b/tests/ui/threads-sendsync/issue-43733.rs index 935e02944b9..0eadef3e3e8 100644 --- a/tests/ui/threads-sendsync/issue-43733.rs +++ b/tests/ui/threads-sendsync/issue-43733.rs @@ -1,8 +1,8 @@ // ignore-wasm32 // revisions: mir thir // [thir]compile-flags: -Z thir-unsafeck -// normalize-stderr-test: "__FastLocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get" -// normalize-stderr-test: "__OsLocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get" +// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get" +// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get" #![feature(thread_local)] #![feature(cfg_target_thread_local, thread_local_internals)] @@ -12,10 +12,10 @@ type Foo = std::cell::RefCell<String>; #[cfg(target_thread_local)] #[thread_local] -static __KEY: std::thread::__FastLocalKeyInner<Foo> = std::thread::__FastLocalKeyInner::new(); +static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new(); #[cfg(not(target_thread_local))] -static __KEY: std::thread::__OsLocalKeyInner<Foo> = std::thread::__OsLocalKeyInner::new(); +static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new(); fn __getit(_: Option<&mut Option<RefCell<String>>>) -> std::option::Option<&'static Foo> { __KEY.get(Default::default) |