From bfa84842e546a5fb2d1c69ba499ddc94bee1e01a Mon Sep 17 00:00:00 2001 From: Jethro Beekman Date: Fri, 7 May 2021 13:40:43 +0200 Subject: Rearrange SGX split module files In #75979 several inlined modules were split out into multiple files. This PR keeps the multiple files but moves a few things around to organize things in a coherent way. --- library/std/src/sys/sgx/abi/tls.rs | 132 ------------ library/std/src/sys/sgx/abi/tls/mod.rs | 132 ++++++++++++ library/std/src/sys/sgx/waitqueue.rs | 245 ----------------------- library/std/src/sys/sgx/waitqueue/mod.rs | 240 ++++++++++++++++++++++ library/std/src/sys/sgx/waitqueue/spin_mutex.rs | 3 + library/std/src/sys/sgx/waitqueue/unsafe_list.rs | 3 + 6 files changed, 378 insertions(+), 377 deletions(-) delete mode 100644 library/std/src/sys/sgx/abi/tls.rs create mode 100644 library/std/src/sys/sgx/abi/tls/mod.rs delete mode 100644 library/std/src/sys/sgx/waitqueue.rs create mode 100644 library/std/src/sys/sgx/waitqueue/mod.rs (limited to 'library/std/src/sys') diff --git a/library/std/src/sys/sgx/abi/tls.rs b/library/std/src/sys/sgx/abi/tls.rs deleted file mode 100644 index 13d96e9a633..00000000000 --- a/library/std/src/sys/sgx/abi/tls.rs +++ /dev/null @@ -1,132 +0,0 @@ -mod sync_bitset; - -use self::sync_bitset::*; -use crate::cell::Cell; -use crate::mem; -use crate::num::NonZeroUsize; -use crate::ptr; -use crate::sync::atomic::{AtomicUsize, Ordering}; - -#[cfg(target_pointer_width = "64")] -const USIZE_BITS: usize = 64; -const TLS_KEYS: usize = 128; // Same as POSIX minimum -const TLS_KEYS_BITSET_SIZE: usize = (TLS_KEYS + (USIZE_BITS - 1)) / USIZE_BITS; - -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_KEY_IN_USEE"] -static TLS_KEY_IN_USE: SyncBitset = SYNC_BITSET_INIT; -macro_rules! dup { - ((* $($exp:tt)*) $($val:tt)*) => (dup!( ($($exp)*) $($val)* $($val)* )); - (() $($val:tt)*) => ([$($val),*]) -} -#[cfg_attr(test, linkage = "available_externally")] -#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_DESTRUCTORE"] -static TLS_DESTRUCTOR: [AtomicUsize; TLS_KEYS] = dup!((* * * * * * *) (AtomicUsize::new(0))); - -extern "C" { - fn get_tls_ptr() -> *const u8; - fn set_tls_ptr(tls: *const u8); -} - -#[derive(Copy, Clone)] -#[repr(C)] -pub struct Key(NonZeroUsize); - -impl Key { - fn to_index(self) -> usize { - self.0.get() - 1 - } - - fn from_index(index: usize) -> Self { - Key(NonZeroUsize::new(index + 1).unwrap()) - } - - pub fn as_usize(self) -> usize { - self.0.get() - } - - pub fn from_usize(index: usize) -> Self { - Key(NonZeroUsize::new(index).unwrap()) - } -} - -#[repr(C)] -pub struct Tls { - data: [Cell<*mut u8>; TLS_KEYS], -} - -pub struct ActiveTls<'a> { - tls: &'a Tls, -} - -impl<'a> Drop for ActiveTls<'a> { - fn drop(&mut self) { - let value_with_destructor = |key: usize| { - let ptr = TLS_DESTRUCTOR[key].load(Ordering::Relaxed); - unsafe { mem::transmute::<_, Option>(ptr) } - .map(|dtor| (&self.tls.data[key], dtor)) - }; - - let mut any_non_null_dtor = true; - while any_non_null_dtor { - any_non_null_dtor = false; - for (value, dtor) in TLS_KEY_IN_USE.iter().filter_map(&value_with_destructor) { - let value = value.replace(ptr::null_mut()); - if !value.is_null() { - any_non_null_dtor = true; - unsafe { dtor(value) } - } - } - } - } -} - -impl Tls { - pub fn new() -> Tls { - Tls { data: dup!((* * * * * * *) (Cell::new(ptr::null_mut()))) } - } - - pub unsafe fn activate(&self) -> ActiveTls<'_> { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { set_tls_ptr(self as *const Tls as _) }; - ActiveTls { tls: self } - } - - #[allow(unused)] - pub unsafe fn activate_persistent(self: Box) { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { set_tls_ptr((&*self) as *const Tls as _) }; - mem::forget(self); - } - - unsafe fn current<'a>() -> &'a Tls { - // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. - unsafe { &*(get_tls_ptr() as *const Tls) } - } - - pub fn create(dtor: Option) -> Key { - let index = if let Some(index) = TLS_KEY_IN_USE.set() { - index - } else { - rtabort!("TLS limit exceeded") - }; - TLS_DESTRUCTOR[index].store(dtor.map_or(0, |f| f as usize), Ordering::Relaxed); - Key::from_index(index) - } - - pub fn set(key: Key, value: *mut u8) { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].set(value); - } - - pub fn get(key: Key) -> *mut u8 { - let index = key.to_index(); - rtassert!(TLS_KEY_IN_USE.get(index)); - unsafe { Self::current() }.data[index].get() - } - - pub fn destroy(key: Key) { - TLS_KEY_IN_USE.clear(key.to_index()); - } -} diff --git a/library/std/src/sys/sgx/abi/tls/mod.rs b/library/std/src/sys/sgx/abi/tls/mod.rs new file mode 100644 index 00000000000..13d96e9a633 --- /dev/null +++ b/library/std/src/sys/sgx/abi/tls/mod.rs @@ -0,0 +1,132 @@ +mod sync_bitset; + +use self::sync_bitset::*; +use crate::cell::Cell; +use crate::mem; +use crate::num::NonZeroUsize; +use crate::ptr; +use crate::sync::atomic::{AtomicUsize, Ordering}; + +#[cfg(target_pointer_width = "64")] +const USIZE_BITS: usize = 64; +const TLS_KEYS: usize = 128; // Same as POSIX minimum +const TLS_KEYS_BITSET_SIZE: usize = (TLS_KEYS + (USIZE_BITS - 1)) / USIZE_BITS; + +#[cfg_attr(test, linkage = "available_externally")] +#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_KEY_IN_USEE"] +static TLS_KEY_IN_USE: SyncBitset = SYNC_BITSET_INIT; +macro_rules! dup { + ((* $($exp:tt)*) $($val:tt)*) => (dup!( ($($exp)*) $($val)* $($val)* )); + (() $($val:tt)*) => ([$($val),*]) +} +#[cfg_attr(test, linkage = "available_externally")] +#[export_name = "_ZN16__rust_internals3std3sys3sgx3abi3tls14TLS_DESTRUCTORE"] +static TLS_DESTRUCTOR: [AtomicUsize; TLS_KEYS] = dup!((* * * * * * *) (AtomicUsize::new(0))); + +extern "C" { + fn get_tls_ptr() -> *const u8; + fn set_tls_ptr(tls: *const u8); +} + +#[derive(Copy, Clone)] +#[repr(C)] +pub struct Key(NonZeroUsize); + +impl Key { + fn to_index(self) -> usize { + self.0.get() - 1 + } + + fn from_index(index: usize) -> Self { + Key(NonZeroUsize::new(index + 1).unwrap()) + } + + pub fn as_usize(self) -> usize { + self.0.get() + } + + pub fn from_usize(index: usize) -> Self { + Key(NonZeroUsize::new(index).unwrap()) + } +} + +#[repr(C)] +pub struct Tls { + data: [Cell<*mut u8>; TLS_KEYS], +} + +pub struct ActiveTls<'a> { + tls: &'a Tls, +} + +impl<'a> Drop for ActiveTls<'a> { + fn drop(&mut self) { + let value_with_destructor = |key: usize| { + let ptr = TLS_DESTRUCTOR[key].load(Ordering::Relaxed); + unsafe { mem::transmute::<_, Option>(ptr) } + .map(|dtor| (&self.tls.data[key], dtor)) + }; + + let mut any_non_null_dtor = true; + while any_non_null_dtor { + any_non_null_dtor = false; + for (value, dtor) in TLS_KEY_IN_USE.iter().filter_map(&value_with_destructor) { + let value = value.replace(ptr::null_mut()); + if !value.is_null() { + any_non_null_dtor = true; + unsafe { dtor(value) } + } + } + } + } +} + +impl Tls { + pub fn new() -> Tls { + Tls { data: dup!((* * * * * * *) (Cell::new(ptr::null_mut()))) } + } + + pub unsafe fn activate(&self) -> ActiveTls<'_> { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { set_tls_ptr(self as *const Tls as _) }; + ActiveTls { tls: self } + } + + #[allow(unused)] + pub unsafe fn activate_persistent(self: Box) { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { set_tls_ptr((&*self) as *const Tls as _) }; + mem::forget(self); + } + + unsafe fn current<'a>() -> &'a Tls { + // FIXME: Needs safety information. See entry.S for `set_tls_ptr` definition. + unsafe { &*(get_tls_ptr() as *const Tls) } + } + + pub fn create(dtor: Option) -> Key { + let index = if let Some(index) = TLS_KEY_IN_USE.set() { + index + } else { + rtabort!("TLS limit exceeded") + }; + TLS_DESTRUCTOR[index].store(dtor.map_or(0, |f| f as usize), Ordering::Relaxed); + Key::from_index(index) + } + + pub fn set(key: Key, value: *mut u8) { + let index = key.to_index(); + rtassert!(TLS_KEY_IN_USE.get(index)); + unsafe { Self::current() }.data[index].set(value); + } + + pub fn get(key: Key) -> *mut u8 { + let index = key.to_index(); + rtassert!(TLS_KEY_IN_USE.get(index)); + unsafe { Self::current() }.data[index].get() + } + + pub fn destroy(key: Key) { + TLS_KEY_IN_USE.clear(key.to_index()); + } +} diff --git a/library/std/src/sys/sgx/waitqueue.rs b/library/std/src/sys/sgx/waitqueue.rs deleted file mode 100644 index e464dc3ee9d..00000000000 --- a/library/std/src/sys/sgx/waitqueue.rs +++ /dev/null @@ -1,245 +0,0 @@ -//! A simple queue implementation for synchronization primitives. -//! -//! This queue is used to implement condition variable and mutexes. -//! -//! Users of this API are expected to use the `WaitVariable` type. Since -//! that type is not `Sync`, it needs to be protected by e.g., a `SpinMutex` to -//! allow shared access. -//! -//! Since userspace may send spurious wake-ups, the wakeup event state is -//! recorded in the enclave. The wakeup event state is protected by a spinlock. -//! The queue and associated wait state are stored in a `WaitVariable`. - -#[cfg(test)] -mod tests; - -/// A doubly-linked list where callers are in charge of memory allocation -/// of the nodes in the list. -mod unsafe_list; - -/// Trivial spinlock-based implementation of `sync::Mutex`. -// FIXME: Perhaps use Intel TSX to avoid locking? -mod spin_mutex; - -use crate::num::NonZeroUsize; -use crate::ops::{Deref, DerefMut}; -use crate::time::Duration; - -use super::abi::thread; -use super::abi::usercalls; -use fortanix_sgx_abi::{Tcs, EV_UNPARK, WAIT_INDEFINITE}; - -pub use self::spin_mutex::{try_lock_or_false, SpinMutex, SpinMutexGuard}; -use self::unsafe_list::{UnsafeList, UnsafeListEntry}; - -/// An queue entry in a `WaitQueue`. -struct WaitEntry { - /// TCS address of the thread that is waiting - tcs: Tcs, - /// Whether this thread has been notified to be awoken - wake: bool, -} - -/// Data stored with a `WaitQueue` alongside it. This ensures accesses to the -/// queue and the data are synchronized, since the type itself is not `Sync`. -/// -/// Consumers of this API should use a synchronization primitive for shared -/// access, such as `SpinMutex`. -#[derive(Default)] -pub struct WaitVariable { - queue: WaitQueue, - lock: T, -} - -impl WaitVariable { - pub const fn new(var: T) -> Self { - WaitVariable { queue: WaitQueue::new(), lock: var } - } - - pub fn queue_empty(&self) -> bool { - self.queue.is_empty() - } - - pub fn lock_var(&self) -> &T { - &self.lock - } - - pub fn lock_var_mut(&mut self) -> &mut T { - &mut self.lock - } -} - -#[derive(Copy, Clone)] -pub enum NotifiedTcs { - Single(Tcs), - All { count: NonZeroUsize }, -} - -/// An RAII guard that will notify a set of target threads as well as unlock -/// a mutex on drop. -pub struct WaitGuard<'a, T: 'a> { - mutex_guard: Option>>, - notified_tcs: NotifiedTcs, -} - -/// A queue of threads that are waiting on some synchronization primitive. -/// -/// `UnsafeList` entries are allocated on the waiting thread's stack. This -/// avoids any global locking that might happen in the heap allocator. This is -/// safe because the waiting thread will not return from that stack frame until -/// after it is notified. The notifying thread ensures to clean up any -/// references to the list entries before sending the wakeup event. -pub struct WaitQueue { - // We use an inner Mutex here to protect the data in the face of spurious - // wakeups. - inner: UnsafeList>, -} -unsafe impl Send for WaitQueue {} - -impl Default for WaitQueue { - fn default() -> Self { - Self::new() - } -} - -impl<'a, T> WaitGuard<'a, T> { - /// Returns which TCSes will be notified when this guard drops. - pub fn notified_tcs(&self) -> NotifiedTcs { - self.notified_tcs - } - - /// Drop this `WaitGuard`, after dropping another `guard`. - pub fn drop_after(self, guard: U) { - drop(guard); - drop(self); - } -} - -impl<'a, T> Deref for WaitGuard<'a, T> { - type Target = SpinMutexGuard<'a, WaitVariable>; - - fn deref(&self) -> &Self::Target { - self.mutex_guard.as_ref().unwrap() - } -} - -impl<'a, T> DerefMut for WaitGuard<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.mutex_guard.as_mut().unwrap() - } -} - -impl<'a, T> Drop for WaitGuard<'a, T> { - fn drop(&mut self) { - drop(self.mutex_guard.take()); - let target_tcs = match self.notified_tcs { - NotifiedTcs::Single(tcs) => Some(tcs), - NotifiedTcs::All { .. } => None, - }; - rtunwrap!(Ok, usercalls::send(EV_UNPARK, target_tcs)); - } -} - -impl WaitQueue { - pub const fn new() -> Self { - WaitQueue { inner: UnsafeList::new() } - } - - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } - - /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait - /// until a wakeup event. - /// - /// This function does not return until this thread has been awoken. - pub fn wait(mut guard: SpinMutexGuard<'_, WaitVariable>, before_wait: F) { - // very unsafe: check requirements of UnsafeList::push - unsafe { - let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { - tcs: thread::current(), - wake: false, - })); - let entry = guard.queue.inner.push(&mut entry); - drop(guard); - before_wait(); - while !entry.lock().wake { - // don't panic, this would invalidate `entry` during unwinding - let eventset = rtunwrap!(Ok, usercalls::wait(EV_UNPARK, WAIT_INDEFINITE)); - rtassert!(eventset & EV_UNPARK == EV_UNPARK); - } - } - } - - /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait - /// until a wakeup event or timeout. If event was observed, returns true. - /// If not, it will remove the calling thread from the wait queue. - pub fn wait_timeout( - lock: &SpinMutex>, - timeout: Duration, - before_wait: F, - ) -> bool { - // very unsafe: check requirements of UnsafeList::push - unsafe { - let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { - tcs: thread::current(), - wake: false, - })); - let entry_lock = lock.lock().queue.inner.push(&mut entry); - before_wait(); - usercalls::wait_timeout(EV_UNPARK, timeout, || entry_lock.lock().wake); - // acquire the wait queue's lock first to avoid deadlock. - let mut guard = lock.lock(); - let success = entry_lock.lock().wake; - if !success { - // nobody is waking us up, so remove our entry from the wait queue. - guard.queue.inner.remove(&mut entry); - } - success - } - } - - /// Either find the next waiter on the wait queue, or return the mutex - /// guard unchanged. - /// - /// If a waiter is found, a `WaitGuard` is returned which will notify the - /// waiter when it is dropped. - pub fn notify_one( - mut guard: SpinMutexGuard<'_, WaitVariable>, - ) -> Result, SpinMutexGuard<'_, WaitVariable>> { - unsafe { - if let Some(entry) = guard.queue.inner.pop() { - let mut entry_guard = entry.lock(); - let tcs = entry_guard.tcs; - entry_guard.wake = true; - drop(entry); - Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::Single(tcs) }) - } else { - Err(guard) - } - } - } - - /// Either find any and all waiters on the wait queue, or return the mutex - /// guard unchanged. - /// - /// If at least one waiter is found, a `WaitGuard` is returned which will - /// notify all waiters when it is dropped. - pub fn notify_all( - mut guard: SpinMutexGuard<'_, WaitVariable>, - ) -> Result, SpinMutexGuard<'_, WaitVariable>> { - unsafe { - let mut count = 0; - while let Some(entry) = guard.queue.inner.pop() { - count += 1; - let mut entry_guard = entry.lock(); - entry_guard.wake = true; - } - if let Some(count) = NonZeroUsize::new(count) { - Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::All { count } }) - } else { - Err(guard) - } - } - } -} diff --git a/library/std/src/sys/sgx/waitqueue/mod.rs b/library/std/src/sys/sgx/waitqueue/mod.rs new file mode 100644 index 00000000000..61bb11d9a6f --- /dev/null +++ b/library/std/src/sys/sgx/waitqueue/mod.rs @@ -0,0 +1,240 @@ +//! A simple queue implementation for synchronization primitives. +//! +//! This queue is used to implement condition variable and mutexes. +//! +//! Users of this API are expected to use the `WaitVariable` type. Since +//! that type is not `Sync`, it needs to be protected by e.g., a `SpinMutex` to +//! allow shared access. +//! +//! Since userspace may send spurious wake-ups, the wakeup event state is +//! recorded in the enclave. The wakeup event state is protected by a spinlock. +//! The queue and associated wait state are stored in a `WaitVariable`. + +#[cfg(test)] +mod tests; + +mod spin_mutex; +mod unsafe_list; + +use crate::num::NonZeroUsize; +use crate::ops::{Deref, DerefMut}; +use crate::time::Duration; + +use super::abi::thread; +use super::abi::usercalls; +use fortanix_sgx_abi::{Tcs, EV_UNPARK, WAIT_INDEFINITE}; + +pub use self::spin_mutex::{try_lock_or_false, SpinMutex, SpinMutexGuard}; +use self::unsafe_list::{UnsafeList, UnsafeListEntry}; + +/// An queue entry in a `WaitQueue`. +struct WaitEntry { + /// TCS address of the thread that is waiting + tcs: Tcs, + /// Whether this thread has been notified to be awoken + wake: bool, +} + +/// Data stored with a `WaitQueue` alongside it. This ensures accesses to the +/// queue and the data are synchronized, since the type itself is not `Sync`. +/// +/// Consumers of this API should use a synchronization primitive for shared +/// access, such as `SpinMutex`. +#[derive(Default)] +pub struct WaitVariable { + queue: WaitQueue, + lock: T, +} + +impl WaitVariable { + pub const fn new(var: T) -> Self { + WaitVariable { queue: WaitQueue::new(), lock: var } + } + + pub fn queue_empty(&self) -> bool { + self.queue.is_empty() + } + + pub fn lock_var(&self) -> &T { + &self.lock + } + + pub fn lock_var_mut(&mut self) -> &mut T { + &mut self.lock + } +} + +#[derive(Copy, Clone)] +pub enum NotifiedTcs { + Single(Tcs), + All { count: NonZeroUsize }, +} + +/// An RAII guard that will notify a set of target threads as well as unlock +/// a mutex on drop. +pub struct WaitGuard<'a, T: 'a> { + mutex_guard: Option>>, + notified_tcs: NotifiedTcs, +} + +/// A queue of threads that are waiting on some synchronization primitive. +/// +/// `UnsafeList` entries are allocated on the waiting thread's stack. This +/// avoids any global locking that might happen in the heap allocator. This is +/// safe because the waiting thread will not return from that stack frame until +/// after it is notified. The notifying thread ensures to clean up any +/// references to the list entries before sending the wakeup event. +pub struct WaitQueue { + // We use an inner Mutex here to protect the data in the face of spurious + // wakeups. + inner: UnsafeList>, +} +unsafe impl Send for WaitQueue {} + +impl Default for WaitQueue { + fn default() -> Self { + Self::new() + } +} + +impl<'a, T> WaitGuard<'a, T> { + /// Returns which TCSes will be notified when this guard drops. + pub fn notified_tcs(&self) -> NotifiedTcs { + self.notified_tcs + } + + /// Drop this `WaitGuard`, after dropping another `guard`. + pub fn drop_after(self, guard: U) { + drop(guard); + drop(self); + } +} + +impl<'a, T> Deref for WaitGuard<'a, T> { + type Target = SpinMutexGuard<'a, WaitVariable>; + + fn deref(&self) -> &Self::Target { + self.mutex_guard.as_ref().unwrap() + } +} + +impl<'a, T> DerefMut for WaitGuard<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.mutex_guard.as_mut().unwrap() + } +} + +impl<'a, T> Drop for WaitGuard<'a, T> { + fn drop(&mut self) { + drop(self.mutex_guard.take()); + let target_tcs = match self.notified_tcs { + NotifiedTcs::Single(tcs) => Some(tcs), + NotifiedTcs::All { .. } => None, + }; + rtunwrap!(Ok, usercalls::send(EV_UNPARK, target_tcs)); + } +} + +impl WaitQueue { + pub const fn new() -> Self { + WaitQueue { inner: UnsafeList::new() } + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait + /// until a wakeup event. + /// + /// This function does not return until this thread has been awoken. + pub fn wait(mut guard: SpinMutexGuard<'_, WaitVariable>, before_wait: F) { + // very unsafe: check requirements of UnsafeList::push + unsafe { + let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { + tcs: thread::current(), + wake: false, + })); + let entry = guard.queue.inner.push(&mut entry); + drop(guard); + before_wait(); + while !entry.lock().wake { + // don't panic, this would invalidate `entry` during unwinding + let eventset = rtunwrap!(Ok, usercalls::wait(EV_UNPARK, WAIT_INDEFINITE)); + rtassert!(eventset & EV_UNPARK == EV_UNPARK); + } + } + } + + /// Adds the calling thread to the `WaitVariable`'s wait queue, then wait + /// until a wakeup event or timeout. If event was observed, returns true. + /// If not, it will remove the calling thread from the wait queue. + pub fn wait_timeout( + lock: &SpinMutex>, + timeout: Duration, + before_wait: F, + ) -> bool { + // very unsafe: check requirements of UnsafeList::push + unsafe { + let mut entry = UnsafeListEntry::new(SpinMutex::new(WaitEntry { + tcs: thread::current(), + wake: false, + })); + let entry_lock = lock.lock().queue.inner.push(&mut entry); + before_wait(); + usercalls::wait_timeout(EV_UNPARK, timeout, || entry_lock.lock().wake); + // acquire the wait queue's lock first to avoid deadlock. + let mut guard = lock.lock(); + let success = entry_lock.lock().wake; + if !success { + // nobody is waking us up, so remove our entry from the wait queue. + guard.queue.inner.remove(&mut entry); + } + success + } + } + + /// Either find the next waiter on the wait queue, or return the mutex + /// guard unchanged. + /// + /// If a waiter is found, a `WaitGuard` is returned which will notify the + /// waiter when it is dropped. + pub fn notify_one( + mut guard: SpinMutexGuard<'_, WaitVariable>, + ) -> Result, SpinMutexGuard<'_, WaitVariable>> { + unsafe { + if let Some(entry) = guard.queue.inner.pop() { + let mut entry_guard = entry.lock(); + let tcs = entry_guard.tcs; + entry_guard.wake = true; + drop(entry); + Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::Single(tcs) }) + } else { + Err(guard) + } + } + } + + /// Either find any and all waiters on the wait queue, or return the mutex + /// guard unchanged. + /// + /// If at least one waiter is found, a `WaitGuard` is returned which will + /// notify all waiters when it is dropped. + pub fn notify_all( + mut guard: SpinMutexGuard<'_, WaitVariable>, + ) -> Result, SpinMutexGuard<'_, WaitVariable>> { + unsafe { + let mut count = 0; + while let Some(entry) = guard.queue.inner.pop() { + count += 1; + let mut entry_guard = entry.lock(); + entry_guard.wake = true; + } + if let Some(count) = NonZeroUsize::new(count) { + Ok(WaitGuard { mutex_guard: Some(guard), notified_tcs: NotifiedTcs::All { count } }) + } else { + Err(guard) + } + } + } +} diff --git a/library/std/src/sys/sgx/waitqueue/spin_mutex.rs b/library/std/src/sys/sgx/waitqueue/spin_mutex.rs index 7f1a671bab4..f6e851ccadd 100644 --- a/library/std/src/sys/sgx/waitqueue/spin_mutex.rs +++ b/library/std/src/sys/sgx/waitqueue/spin_mutex.rs @@ -1,3 +1,6 @@ +//! Trivial spinlock-based implementation of `sync::Mutex`. +// FIXME: Perhaps use Intel TSX to avoid locking? + #[cfg(test)] mod tests; diff --git a/library/std/src/sys/sgx/waitqueue/unsafe_list.rs b/library/std/src/sys/sgx/waitqueue/unsafe_list.rs index 0834d2593fc..cf2f0886c15 100644 --- a/library/std/src/sys/sgx/waitqueue/unsafe_list.rs +++ b/library/std/src/sys/sgx/waitqueue/unsafe_list.rs @@ -1,3 +1,6 @@ +//! A doubly-linked list where callers are in charge of memory allocation +//! of the nodes in the list. + #[cfg(test)] mod tests; -- cgit 1.4.1-3-g733a5