std: move `Once` implementations to `sys`

41 files changed, 488 insertions, 25 deletions

diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs
index 81200e0061e..bbd1d840e92 100644
--- a/library/std/src/sys/mod.rs
+++ b/library/std/src/sys/mod.rs
@@ -6,9 +6,9 @@ mod pal;
 mod personality;
 
 pub mod cmath;
-pub mod locks;
 pub mod os_str;
 pub mod path;
+pub mod sync;
 #[allow(dead_code)]
 #[allow(unused_imports)]
 pub mod thread_local;
diff --git a/library/std/src/sys/pal/teeos/mod.rs b/library/std/src/sys/pal/teeos/mod.rs
index 1fb9d5438de..c392a0ea264 100644
--- a/library/std/src/sys/pal/teeos/mod.rs
+++ b/library/std/src/sys/pal/teeos/mod.rs
@@ -19,8 +19,6 @@ pub mod fs;
 #[path = "../unsupported/io.rs"]
 pub mod io;
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/pal/uefi/mod.rs b/library/std/src/sys/pal/uefi/mod.rs
index 7c5b37fb490..562b00c2c01 100644
--- a/library/std/src/sys/pal/uefi/mod.rs
+++ b/library/std/src/sys/pal/uefi/mod.rs
@@ -21,8 +21,6 @@ pub mod fs;
 pub mod io;
 #[path = "../unsupported/net.rs"]
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/pal/unsupported/mod.rs b/library/std/src/sys/pal/unsupported/mod.rs
index 9ce275ee72d..be344fb7cae 100644
--- a/library/std/src/sys/pal/unsupported/mod.rs
+++ b/library/std/src/sys/pal/unsupported/mod.rs
@@ -6,7 +6,6 @@ pub mod env;
 pub mod fs;
 pub mod io;
 pub mod net;
-pub mod once;
 pub mod os;
 pub mod pipe;
 pub mod process;
diff --git a/library/std/src/sys/pal/wasi/mod.rs b/library/std/src/sys/pal/wasi/mod.rs
index 308dd296004..a78547261ad 100644
--- a/library/std/src/sys/pal/wasi/mod.rs
+++ b/library/std/src/sys/pal/wasi/mod.rs
@@ -41,8 +41,6 @@ pub mod time;
 
 cfg_if::cfg_if! {
     if #[cfg(not(target_feature = "atomics"))] {
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread_parking.rs"]
         pub mod thread_parking;
     }
diff --git a/library/std/src/sys/pal/wasip2/mod.rs b/library/std/src/sys/pal/wasip2/mod.rs
index b12a8d5ea11..d1d444d7b79 100644
--- a/library/std/src/sys/pal/wasip2/mod.rs
+++ b/library/std/src/sys/pal/wasip2/mod.rs
@@ -51,10 +51,6 @@ cfg_if::cfg_if! {
     if #[cfg(target_feature = "atomics")] {
         compile_error!("The wasm32-wasip2 target does not support atomics");
     } else {
-        #[path = "../unsupported/locks/mod.rs"]
-        pub mod locks;
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread_parking.rs"]
         pub mod thread_parking;
     }
diff --git a/library/std/src/sys/pal/wasm/mod.rs b/library/std/src/sys/pal/wasm/mod.rs
index 40b15120e6d..5cbc3e45341 100644
--- a/library/std/src/sys/pal/wasm/mod.rs
+++ b/library/std/src/sys/pal/wasm/mod.rs
@@ -48,8 +48,6 @@ cfg_if::cfg_if! {
         #[path = "atomics/thread.rs"]
         pub mod thread;
     } else {
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread.rs"]
         pub mod thread;
         #[path = "../unsupported/thread_parking.rs"]
diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs
index 6c714f76309..228a976dbab 100644
--- a/library/std/src/sys/pal/zkvm/mod.rs
+++ b/library/std/src/sys/pal/zkvm/mod.rs
@@ -21,8 +21,6 @@ pub mod fs;
 pub mod io;
 #[path = "../unsupported/net.rs"]
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/locks/condvar/futex.rs b/library/std/src/sys/sync/condvar/futex.rs
index 3ad93ce07f7..4586d0fd941 100644
--- a/library/std/src/sys/locks/condvar/futex.rs
+++ b/library/std/src/sys/sync/condvar/futex.rs
@@ -1,6 +1,6 @@
 use crate::sync::atomic::{AtomicU32, Ordering::Relaxed};
 use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 
 pub struct Condvar {
diff --git a/library/std/src/sys/locks/condvar/itron.rs b/library/std/src/sys/sync/condvar/itron.rs
index 4c6f5e9dad2..9b64d241efd 100644
--- a/library/std/src/sys/locks/condvar/itron.rs
+++ b/library/std/src/sys/sync/condvar/itron.rs
@@ -2,7 +2,7 @@
 use crate::sys::pal::itron::{
     abi, error::expect_success_aborting, spin::SpinMutex, task, time::with_tmos_strong,
 };
-use crate::{mem::replace, ptr::NonNull, sys::locks::Mutex, time::Duration};
+use crate::{mem::replace, ptr::NonNull, sys::sync::Mutex, time::Duration};
 
 // The implementation is inspired by the queue-based implementation shown in
 // Andrew D. Birrell's paper "Implementing Condition Variables with Semaphores"
diff --git a/library/std/src/sys/locks/condvar/mod.rs b/library/std/src/sys/sync/condvar/mod.rs
index 6849cacf88e..6849cacf88e 100644
--- a/library/std/src/sys/locks/condvar/mod.rs
+++ b/library/std/src/sys/sync/condvar/mod.rs
diff --git a/library/std/src/sys/locks/condvar/no_threads.rs b/library/std/src/sys/sync/condvar/no_threads.rs
index 3f0943b50ee..36b89c5f5be 100644
--- a/library/std/src/sys/locks/condvar/no_threads.rs
+++ b/library/std/src/sys/sync/condvar/no_threads.rs
@@ -1,4 +1,4 @@
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 
 pub struct Condvar {}
diff --git a/library/std/src/sys/locks/condvar/pthread.rs b/library/std/src/sys/sync/condvar/pthread.rs
index 094738d5a3f..728371685ee 100644
--- a/library/std/src/sys/locks/condvar/pthread.rs
+++ b/library/std/src/sys/sync/condvar/pthread.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
 use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
-use crate::sys::locks::{mutex, Mutex};
+use crate::sys::sync::{mutex, Mutex};
 #[cfg(not(target_os = "nto"))]
 use crate::sys::time::TIMESPEC_MAX;
 #[cfg(target_os = "nto")]
diff --git a/library/std/src/sys/locks/condvar/sgx.rs b/library/std/src/sys/sync/condvar/sgx.rs
index cabd3250275..ecb5872f60d 100644
--- a/library/std/src/sys/locks/condvar/sgx.rs
+++ b/library/std/src/sys/sync/condvar/sgx.rs
@@ -1,5 +1,5 @@
-use crate::sys::locks::Mutex;
 use crate::sys::pal::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
+use crate::sys::sync::Mutex;
 use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
 
diff --git a/library/std/src/sys/locks/condvar/teeos.rs b/library/std/src/sys/sync/condvar/teeos.rs
index c08e8145b8c..0a931f407d2 100644
--- a/library/std/src/sys/locks/condvar/teeos.rs
+++ b/library/std/src/sys/sync/condvar/teeos.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
 use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
-use crate::sys::locks::mutex::{self, Mutex};
+use crate::sys::sync::mutex::{self, Mutex};
 use crate::sys::time::TIMESPEC_MAX;
 use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
diff --git a/library/std/src/sys/locks/condvar/windows7.rs b/library/std/src/sys/sync/condvar/windows7.rs
index 28a288335d2..07fa5fdd698 100644
--- a/library/std/src/sys/locks/condvar/windows7.rs
+++ b/library/std/src/sys/sync/condvar/windows7.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::sys::c;
-use crate::sys::locks::{mutex, Mutex};
 use crate::sys::os;
+use crate::sys::sync::{mutex, Mutex};
 use crate::time::Duration;
 
 pub struct Condvar {
diff --git a/library/std/src/sys/locks/condvar/xous.rs b/library/std/src/sys/sync/condvar/xous.rs
index 0e51449e0af..7b218818ef8 100644
--- a/library/std/src/sys/locks/condvar/xous.rs
+++ b/library/std/src/sys/sync/condvar/xous.rs
@@ -1,6 +1,6 @@
 use crate::os::xous::ffi::{blocking_scalar, scalar};
 use crate::os::xous::services::{ticktimer_server, TicktimerScalar};
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 use core::sync::atomic::{AtomicUsize, Ordering};
 
diff --git a/library/std/src/sys/locks/mod.rs b/library/std/src/sys/sync/mod.rs
index 0bdc4a1e1db..623e6bccd51 100644
--- a/library/std/src/sys/locks/mod.rs
+++ b/library/std/src/sys/sync/mod.rs
@@ -1,7 +1,9 @@
 mod condvar;
 mod mutex;
+mod once;
 mod rwlock;
 
 pub use condvar::Condvar;
 pub use mutex::Mutex;
+pub use once::{Once, OnceState};
 pub use rwlock::RwLock;
diff --git a/library/std/src/sys/locks/mutex/fuchsia.rs b/library/std/src/sys/sync/mutex/fuchsia.rs
index 5d89e5a13fd..5d89e5a13fd 100644
--- a/library/std/src/sys/locks/mutex/fuchsia.rs
+++ b/library/std/src/sys/sync/mutex/fuchsia.rs
diff --git a/library/std/src/sys/locks/mutex/futex.rs b/library/std/src/sys/sync/mutex/futex.rs
index 7427cae94d6..7427cae94d6 100644
--- a/library/std/src/sys/locks/mutex/futex.rs
+++ b/library/std/src/sys/sync/mutex/futex.rs
diff --git a/library/std/src/sys/locks/mutex/itron.rs b/library/std/src/sys/sync/mutex/itron.rs
index a134eb2d1be..a134eb2d1be 100644
--- a/library/std/src/sys/locks/mutex/itron.rs
+++ b/library/std/src/sys/sync/mutex/itron.rs
diff --git a/library/std/src/sys/locks/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs
index 73d9bd273de..73d9bd273de 100644
--- a/library/std/src/sys/locks/mutex/mod.rs
+++ b/library/std/src/sys/sync/mutex/mod.rs
diff --git a/library/std/src/sys/locks/mutex/no_threads.rs b/library/std/src/sys/sync/mutex/no_threads.rs
index 4a13c55fb8b..4a13c55fb8b 100644
--- a/library/std/src/sys/locks/mutex/no_threads.rs
+++ b/library/std/src/sys/sync/mutex/no_threads.rs
diff --git a/library/std/src/sys/locks/mutex/pthread.rs b/library/std/src/sys/sync/mutex/pthread.rs
index ee0794334fb..ee0794334fb 100644
--- a/library/std/src/sys/locks/mutex/pthread.rs
+++ b/library/std/src/sys/sync/mutex/pthread.rs
diff --git a/library/std/src/sys/locks/mutex/sgx.rs b/library/std/src/sys/sync/mutex/sgx.rs
index d37bd02adf8..d37bd02adf8 100644
--- a/library/std/src/sys/locks/mutex/sgx.rs
+++ b/library/std/src/sys/sync/mutex/sgx.rs
diff --git a/library/std/src/sys/locks/mutex/windows7.rs b/library/std/src/sys/sync/mutex/windows7.rs
index ef2f84082cd..ef2f84082cd 100644
--- a/library/std/src/sys/locks/mutex/windows7.rs
+++ b/library/std/src/sys/sync/mutex/windows7.rs
diff --git a/library/std/src/sys/locks/mutex/xous.rs b/library/std/src/sys/sync/mutex/xous.rs
index a8c9518ff0b..a8c9518ff0b 100644
--- a/library/std/src/sys/locks/mutex/xous.rs
+++ b/library/std/src/sys/sync/mutex/xous.rs
diff --git a/library/std/src/sys/sync/once/futex.rs b/library/std/src/sys/sync/once/futex.rs
new file mode 100644
index 00000000000..609085dcd47
--- /dev/null
+++ b/library/std/src/sys/sync/once/futex.rs
@@ -0,0 +1,146 @@
+use crate::cell::Cell;
+use crate::sync as public;
+use crate::sync::atomic::{
+    AtomicU32,
+    Ordering::{Acquire, Relaxed, Release},
+};
+use crate::sync::once::ExclusiveState;
+use crate::sys::futex::{futex_wait, futex_wake_all};
+
+// On some platforms, the OS is very nice and handles the waiter queue for us.
+// This means we only need one atomic value with 5 states:
+
+/// No initialization has run yet, and no thread is currently using the Once.
+const INCOMPLETE: u32 = 0;
+/// Some thread has previously attempted to initialize the Once, but it panicked,
+/// so the Once is now poisoned. There are no other threads currently accessing
+/// this Once.
+const POISONED: u32 = 1;
+/// Some thread is currently attempting to run initialization. It may succeed,
+/// so all future threads need to wait for it to finish.
+const RUNNING: u32 = 2;
+/// Some thread is currently attempting to run initialization and there are threads
+/// waiting for it to finish.
+const QUEUED: u32 = 3;
+/// Initialization has completed and all future calls should finish immediately.
+const COMPLETE: u32 = 4;
+
+// Threads wait by setting the state to QUEUED and calling `futex_wait` on the state
+// variable. When the running thread finishes, it will wake all waiting threads using
+// `futex_wake_all`.
+
+pub struct OnceState {
+    poisoned: bool,
+    set_state_to: Cell<u32>,
+}
+
+impl OnceState {
+    #[inline]
+    pub fn is_poisoned(&self) -> bool {
+        self.poisoned
+    }
+
+    #[inline]
+    pub fn poison(&self) {
+        self.set_state_to.set(POISONED);
+    }
+}
+
+struct CompletionGuard<'a> {
+    state: &'a AtomicU32,
+    set_state_on_drop_to: u32,
+}
+
+impl<'a> Drop for CompletionGuard<'a> {
+    fn drop(&mut self) {
+        // Use release ordering to propagate changes to all threads checking
+        // up on the Once. `futex_wake_all` does its own synchronization, hence
+        // we do not need `AcqRel`.
+        if self.state.swap(self.set_state_on_drop_to, Release) == QUEUED {
+            futex_wake_all(&self.state);
+        }
+    }
+}
+
+pub struct Once {
+    state: AtomicU32,
+}
+
+impl Once {
+    #[inline]
+    pub const fn new() -> Once {
+        Once { state: AtomicU32::new(INCOMPLETE) }
+    }
+
+    #[inline]
+    pub fn is_completed(&self) -> bool {
+        // Use acquire ordering to make all initialization changes visible to the
+        // current thread.
+        self.state.load(Acquire) == COMPLETE
+    }
+
+    #[inline]
+    pub(crate) fn state(&mut self) -> ExclusiveState {
+        match *self.state.get_mut() {
+            INCOMPLETE => ExclusiveState::Incomplete,
+            POISONED => ExclusiveState::Poisoned,
+            COMPLETE => ExclusiveState::Complete,
+            _ => unreachable!("invalid Once state"),
+        }
+    }
+
+    // This uses FnMut to match the API of the generic implementation. As this
+    // implementation is quite light-weight, it is generic over the closure and
+    // so avoids the cost of dynamic dispatch.
+    #[cold]
+    #[track_caller]
+    pub fn call(&self, ignore_poisoning: bool, f: &mut impl FnMut(&public::OnceState)) {
+        let mut state = self.state.load(Acquire);
+        loop {
+            match state {
+                POISONED if !ignore_poisoning => {
+                    // Panic to propagate the poison.
+                    panic!("Once instance has previously been poisoned");
+                }
+                INCOMPLETE | POISONED => {
+                    // Try to register the current thread as the one running.
+                    if let Err(new) =
+                        self.state.compare_exchange_weak(state, RUNNING, Acquire, Acquire)
+                    {
+                        state = new;
+                        continue;
+                    }
+                    // `waiter_queue` will manage other waiting threads, and
+                    // wake them up on drop.
+                    let mut waiter_queue =
+                        CompletionGuard { state: &self.state, set_state_on_drop_to: POISONED };
+                    // Run the function, letting it know if we're poisoned or not.
+                    let f_state = public::OnceState {
+                        inner: OnceState {
+                            poisoned: state == POISONED,
+                            set_state_to: Cell::new(COMPLETE),
+                        },
+                    };
+                    f(&f_state);
+                    waiter_queue.set_state_on_drop_to = f_state.inner.set_state_to.get();
+                    return;
+                }
+                RUNNING | QUEUED => {
+                    // Set the state to QUEUED if it is not already.
+                    if state == RUNNING
+                        && let Err(new) =
+                            self.state.compare_exchange_weak(RUNNING, QUEUED, Relaxed, Acquire)
+                    {
+                        state = new;
+                        continue;
+                    }
+
+                    futex_wait(&self.state, QUEUED, None);
+                    state = self.state.load(Acquire);
+                }
+                COMPLETE => return,
+                _ => unreachable!("state is never set to invalid values"),
+            }
+        }
+    }
+}
diff --git a/library/std/src/sys/sync/once/mod.rs b/library/std/src/sys/sync/once/mod.rs
new file mode 100644
index 00000000000..61b29713fa1
--- /dev/null
+++ b/library/std/src/sys/sync/once/mod.rs
@@ -0,0 +1,36 @@
+// A "once" is a relatively simple primitive, and it's also typically provided
+// by the OS as well (see `pthread_once` or `InitOnceExecuteOnce`). The OS
+// primitives, however, tend to have surprising restrictions, such as the Unix
+// one doesn't allow an argument to be passed to the function.
+//
+// As a result, we end up implementing it ourselves in the standard library.
+// This also gives us the opportunity to optimize the implementation a bit which
+// should help the fast path on call sites.
+
+cfg_if::cfg_if! {
+    if #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        all(target_arch = "wasm32", target_feature = "atomics"),
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "dragonfly",
+        target_os = "fuchsia",
+        target_os = "hermit",
+    ))] {
+        mod futex;
+        pub use futex::{Once, OnceState};
+    } else if #[cfg(any(
+        windows,
+        target_family = "unix",
+        all(target_vendor = "fortanix", target_env = "sgx"),
+        target_os = "solid_asp3",
+        target_os = "xous",
+    ))] {
+        mod queue;
+        pub use queue::{Once, OnceState};
+    } else {
+        mod no_threads;
+        pub use no_threads::{Once, OnceState};
+    }
+}
diff --git a/library/std/src/sys/pal/unsupported/once.rs b/library/std/src/sys/sync/once/no_threads.rs
index 11fde1888ba..11fde1888ba 100644
--- a/library/std/src/sys/pal/unsupported/once.rs
+++ b/library/std/src/sys/sync/once/no_threads.rs
diff --git a/library/std/src/sys/sync/once/queue.rs b/library/std/src/sys/sync/once/queue.rs
new file mode 100644
index 00000000000..730cdb768bd
--- /dev/null
+++ b/library/std/src/sys/sync/once/queue.rs
@@ -0,0 +1,294 @@
+// Each `Once` has one word of atomic state, and this state is CAS'd on to
+// determine what to do. There are four possible state of a `Once`:
+//
+// * Incomplete - no initialization has run yet, and no thread is currently
+//                using the Once.
+// * Poisoned - some thread has previously attempted to initialize the Once, but
+//              it panicked, so the Once is now poisoned. There are no other
+//              threads currently accessing this Once.
+// * Running - some thread is currently attempting to run initialization. It may
+//             succeed, so all future threads need to wait for it to finish.
+//             Note that this state is accompanied with a payload, described
+//             below.
+// * Complete - initialization has completed and all future calls should finish
+//              immediately.
+//
+// With 4 states we need 2 bits to encode this, and we use the remaining bits
+// in the word we have allocated as a queue of threads waiting for the thread
+// responsible for entering the RUNNING state. This queue is just a linked list
+// of Waiter nodes which is monotonically increasing in size. Each node is
+// allocated on the stack, and whenever the running closure finishes it will
+// consume the entire queue and notify all waiters they should try again.
+//
+// You'll find a few more details in the implementation, but that's the gist of
+// it!
+//
+// Atomic orderings:
+// When running `Once` we deal with multiple atomics:
+// `Once.state_and_queue` and an unknown number of `Waiter.signaled`.
+// * `state_and_queue` is used (1) as a state flag, (2) for synchronizing the
+//   result of the `Once`, and (3) for synchronizing `Waiter` nodes.
+//     - At the end of the `call` function we have to make sure the result
+//       of the `Once` is acquired. So every load which can be the only one to
+//       load COMPLETED must have at least acquire ordering, which means all
+//       three of them.
+//     - `WaiterQueue::drop` is the only place that may store COMPLETED, and
+//       must do so with release ordering to make the result available.
+//     - `wait` inserts `Waiter` nodes as a pointer in `state_and_queue`, and
+//       needs to make the nodes available with release ordering. The load in
+//       its `compare_exchange` can be relaxed because it only has to compare
+//       the atomic, not to read other data.
+//     - `WaiterQueue::drop` must see the `Waiter` nodes, so it must load
+//       `state_and_queue` with acquire ordering.
+//     - There is just one store where `state_and_queue` is used only as a
+//       state flag, without having to synchronize data: switching the state
+//       from INCOMPLETE to RUNNING in `call`. This store can be Relaxed,
+//       but the read has to be Acquire because of the requirements mentioned
+//       above.
+// * `Waiter.signaled` is both used as a flag, and to protect a field with
+//   interior mutability in `Waiter`. `Waiter.thread` is changed in
+//   `WaiterQueue::drop` which then sets `signaled` with release ordering.
+//   After `wait` loads `signaled` with acquire ordering and sees it is true,
+//   it needs to see the changes to drop the `Waiter` struct correctly.
+// * There is one place where the two atomics `Once.state_and_queue` and
+//   `Waiter.signaled` come together, and might be reordered by the compiler or
+//   processor. Because both use acquire ordering such a reordering is not
+//   allowed, so no need for `SeqCst`.
+
+use crate::cell::Cell;
+use crate::fmt;
+use crate::ptr;
+use crate::sync as public;
+use crate::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
+use crate::sync::once::ExclusiveState;
+use crate::thread::{self, Thread};
+
+type Masked = ();
+
+pub struct Once {
+    state_and_queue: AtomicPtr<Masked>,
+}
+
+pub struct OnceState {
+    poisoned: bool,
+    set_state_on_drop_to: Cell<*mut Masked>,
+}
+
+// Four states that a Once can be in, encoded into the lower bits of
+// `state_and_queue` in the Once structure.
+const INCOMPLETE: usize = 0x0;
+const POISONED: usize = 0x1;
+const RUNNING: usize = 0x2;
+const COMPLETE: usize = 0x3;
+
+// Mask to learn about the state. All other bits are the queue of waiters if
+// this is in the RUNNING state.
+const STATE_MASK: usize = 0x3;
+
+// Representation of a node in the linked list of waiters, used while in the
+// RUNNING state.
+// Note: `Waiter` can't hold a mutable pointer to the next thread, because then
+// `wait` would both hand out a mutable reference to its `Waiter` node, and keep
+// a shared reference to check `signaled`. Instead we hold shared references and
+// use interior mutability.
+#[repr(align(4))] // Ensure the two lower bits are free to use as state bits.
+struct Waiter {
+    thread: Cell<Option<Thread>>,
+    signaled: AtomicBool,
+    next: *const Waiter,
+}
+
+// Head of a linked list of waiters.
+// Every node is a struct on the stack of a waiting thread.
+// Will wake up the waiters when it gets dropped, i.e. also on panic.
+struct WaiterQueue<'a> {
+    state_and_queue: &'a AtomicPtr<Masked>,
+    set_state_on_drop_to: *mut Masked,
+}
+
+impl Once {
+    #[inline]
+    #[rustc_const_stable(feature = "const_once_new", since = "1.32.0")]
+    pub const fn new() -> Once {
+        Once { state_and_queue: AtomicPtr::new(ptr::without_provenance_mut(INCOMPLETE)) }
+    }
+
+    #[inline]
+    pub fn is_completed(&self) -> bool {
+        // An `Acquire` load is enough because that makes all the initialization
+        // operations visible to us, and, this being a fast path, weaker
+        // ordering helps with performance. This `Acquire` synchronizes with
+        // `Release` operations on the slow path.
+        self.state_and_queue.load(Ordering::Acquire).addr() == COMPLETE
+    }
+
+    #[inline]
+    pub(crate) fn state(&mut self) -> ExclusiveState {
+        match self.state_and_queue.get_mut().addr() {
+            INCOMPLETE => ExclusiveState::Incomplete,
+            POISONED => ExclusiveState::Poisoned,
+            COMPLETE => ExclusiveState::Complete,
+            _ => unreachable!("invalid Once state"),
+        }
+    }
+
+    // This is a non-generic function to reduce the monomorphization cost of
+    // using `call_once` (this isn't exactly a trivial or small implementation).
+    //
+    // Additionally, this is tagged with `#[cold]` as it should indeed be cold
+    // and it helps let LLVM know that calls to this function should be off the
+    // fast path. Essentially, this should help generate more straight line code
+    // in LLVM.
+    //
+    // Finally, this takes an `FnMut` instead of a `FnOnce` because there's
+    // currently no way to take an `FnOnce` and call it via virtual dispatch
+    // without some allocation overhead.
+    #[cold]
+    #[track_caller]
+    pub fn call(&self, ignore_poisoning: bool, init: &mut dyn FnMut(&public::OnceState)) {
+        let mut state_and_queue = self.state_and_queue.load(Ordering::Acquire);
+        loop {
+            match state_and_queue.addr() {
+                COMPLETE => break,
+                POISONED if !ignore_poisoning => {
+                    // Panic to propagate the poison.
+                    panic!("Once instance has previously been poisoned");
+                }
+                POISONED | INCOMPLETE => {
+                    // Try to register this thread as the one RUNNING.
+                    let exchange_result = self.state_and_queue.compare_exchange(
+                        state_and_queue,
+                        ptr::without_provenance_mut(RUNNING),
+                        Ordering::Acquire,
+                        Ordering::Acquire,
+                    );
+                    if let Err(old) = exchange_result {
+                        state_and_queue = old;
+                        continue;
+                    }
+                    // `waiter_queue` will manage other waiting threads, and
+                    // wake them up on drop.
+                    let mut waiter_queue = WaiterQueue {
+                        state_and_queue: &self.state_and_queue,
+                        set_state_on_drop_to: ptr::without_provenance_mut(POISONED),
+                    };
+                    // Run the initialization function, letting it know if we're
+                    // poisoned or not.
+                    let init_state = public::OnceState {
+                        inner: OnceState {
+                            poisoned: state_and_queue.addr() == POISONED,
+                            set_state_on_drop_to: Cell::new(ptr::without_provenance_mut(COMPLETE)),
+                        },
+                    };
+                    init(&init_state);
+                    waiter_queue.set_state_on_drop_to = init_state.inner.set_state_on_drop_to.get();
+                    break;
+                }
+                _ => {
+                    // All other values must be RUNNING with possibly a
+                    // pointer to the waiter queue in the more significant bits.
+                    assert!(state_and_queue.addr() & STATE_MASK == RUNNING);
+                    wait(&self.state_and_queue, state_and_queue);
+                    state_and_queue = self.state_and_queue.load(Ordering::Acquire);
+                }
+            }
+        }
+    }
+}
+
+fn wait(state_and_queue: &AtomicPtr<Masked>, mut current_state: *mut Masked) {
+    // Note: the following code was carefully written to avoid creating a
+    // mutable reference to `node` that gets aliased.
+    loop {
+        // Don't queue this thread if the status is no longer running,
+        // otherwise we will not be woken up.
+        if current_state.addr() & STATE_MASK != RUNNING {
+            return;
+        }
+
+        // Create the node for our current thread.
+        let node = Waiter {
+            thread: Cell::new(Some(thread::current())),
+            signaled: AtomicBool::new(false),
+            next: current_state.with_addr(current_state.addr() & !STATE_MASK) as *const Waiter,
+        };
+        let me = core::ptr::addr_of!(node) as *const Masked as *mut Masked;
+
+        // Try to slide in the node at the head of the linked list, making sure
+        // that another thread didn't just replace the head of the linked list.
+        let exchange_result = state_and_queue.compare_exchange(
+            current_state,
+            me.with_addr(me.addr() | RUNNING),
+            Ordering::Release,
+            Ordering::Relaxed,
+        );
+        if let Err(old) = exchange_result {
+            current_state = old;
+            continue;
+        }
+
+        // We have enqueued ourselves, now lets wait.
+        // It is important not to return before being signaled, otherwise we
+        // would drop our `Waiter` node and leave a hole in the linked list
+        // (and a dangling reference). Guard against spurious wakeups by
+        // reparking ourselves until we are signaled.
+        while !node.signaled.load(Ordering::Acquire) {
+            // If the managing thread happens to signal and unpark us before we
+            // can park ourselves, the result could be this thread never gets
+            // unparked. Luckily `park` comes with the guarantee that if it got
+            // an `unpark` just before on an unparked thread it does not park.
+            thread::park();
+        }
+        break;
+    }
+}
+
+#[stable(feature = "std_debug", since = "1.16.0")]
+impl fmt::Debug for Once {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Once").finish_non_exhaustive()
+    }
+}
+
+impl Drop for WaiterQueue<'_> {
+    fn drop(&mut self) {
+        // Swap out our state with however we finished.
+        let state_and_queue =
+            self.state_and_queue.swap(self.set_state_on_drop_to, Ordering::AcqRel);
+
+        // We should only ever see an old state which was RUNNING.
+        assert_eq!(state_and_queue.addr() & STATE_MASK, RUNNING);
+
+        // Walk the entire linked list of waiters and wake them up (in lifo
+        // order, last to register is first to wake up).
+        unsafe {
+            // Right after setting `node.signaled = true` the other thread may
+            // free `node` if there happens to be has a spurious wakeup.
+            // So we have to take out the `thread` field and copy the pointer to
+            // `next` first.
+            let mut queue =
+                state_and_queue.with_addr(state_and_queue.addr() & !STATE_MASK) as *const Waiter;
+            while !queue.is_null() {
+                let next = (*queue).next;
+                let thread = (*queue).thread.take().unwrap();
+                (*queue).signaled.store(true, Ordering::Release);
+                // ^- FIXME (maybe): This is another case of issue #55005
+                // `store()` has a potentially dangling ref to `signaled`.
+                queue = next;
+                thread.unpark();
+            }
+        }
+    }
+}
+
+impl OnceState {
+    #[inline]
+    pub fn is_poisoned(&self) -> bool {
+        self.poisoned
+    }
+
+    #[inline]
+    pub fn poison(&self) {
+        self.set_state_on_drop_to.set(ptr::without_provenance_mut(POISONED));
+    }
+}
diff --git a/library/std/src/sys/locks/rwlock/futex.rs b/library/std/src/sys/sync/rwlock/futex.rs
index aa0de900238..aa0de900238 100644
--- a/library/std/src/sys/locks/rwlock/futex.rs
+++ b/library/std/src/sys/sync/rwlock/futex.rs
diff --git a/library/std/src/sys/locks/rwlock/mod.rs b/library/std/src/sys/sync/rwlock/mod.rs
index 675931c64bd..675931c64bd 100644
--- a/library/std/src/sys/locks/rwlock/mod.rs
+++ b/library/std/src/sys/sync/rwlock/mod.rs
diff --git a/library/std/src/sys/locks/rwlock/no_threads.rs b/library/std/src/sys/sync/rwlock/no_threads.rs
index 789ef9b29e5..789ef9b29e5 100644
--- a/library/std/src/sys/locks/rwlock/no_threads.rs
+++ b/library/std/src/sys/sync/rwlock/no_threads.rs
diff --git a/library/std/src/sys/locks/rwlock/queue.rs b/library/std/src/sys/sync/rwlock/queue.rs
index dce966086b8..dce966086b8 100644
--- a/library/std/src/sys/locks/rwlock/queue.rs
+++ b/library/std/src/sys/sync/rwlock/queue.rs
diff --git a/library/std/src/sys/locks/rwlock/sgx.rs b/library/std/src/sys/sync/rwlock/sgx.rs
index 136dea597bb..136dea597bb 100644
--- a/library/std/src/sys/locks/rwlock/sgx.rs
+++ b/library/std/src/sys/sync/rwlock/sgx.rs
diff --git a/library/std/src/sys/locks/rwlock/sgx/tests.rs b/library/std/src/sys/sync/rwlock/sgx/tests.rs
index 5fd6670afd4..5fd6670afd4 100644
--- a/library/std/src/sys/locks/rwlock/sgx/tests.rs
+++ b/library/std/src/sys/sync/rwlock/sgx/tests.rs
diff --git a/library/std/src/sys/locks/rwlock/solid.rs b/library/std/src/sys/sync/rwlock/solid.rs
index 9bf6f5dbb73..9bf6f5dbb73 100644
--- a/library/std/src/sys/locks/rwlock/solid.rs
+++ b/library/std/src/sys/sync/rwlock/solid.rs
diff --git a/library/std/src/sys/locks/rwlock/teeos.rs b/library/std/src/sys/sync/rwlock/teeos.rs
index 27cdb88788f..ef9b1ab5154 100644
--- a/library/std/src/sys/locks/rwlock/teeos.rs
+++ b/library/std/src/sys/sync/rwlock/teeos.rs
@@ -1,4 +1,4 @@
-use crate::sys::locks::mutex::Mutex;
+use crate::sys::sync::mutex::Mutex;
 
 /// we do not supported rwlock, so use mutex to simulate rwlock.
 /// it's useful because so many code in std will use rwlock.
diff --git a/library/std/src/sys/locks/rwlock/windows7.rs b/library/std/src/sys/sync/rwlock/windows7.rs
index e69415baac4..e69415baac4 100644
--- a/library/std/src/sys/locks/rwlock/windows7.rs
+++ b/library/std/src/sys/sync/rwlock/windows7.rs
diff --git a/library/std/src/sys/locks/rwlock/xous.rs b/library/std/src/sys/sync/rwlock/xous.rs
index ab45b33e1f6..ab45b33e1f6 100644
--- a/library/std/src/sys/locks/rwlock/xous.rs
+++ b/library/std/src/sys/sync/rwlock/xous.rs