Merge from rust-lang/rust

21 files changed, 240 insertions, 171 deletions

diff --git a/library/std/src/sys/sync/condvar/futex.rs b/library/std/src/sys/sync/condvar/futex.rs
index 4586d0fd941..39cd97c01ea 100644
--- a/library/std/src/sys/sync/condvar/futex.rs
+++ b/library/std/src/sys/sync/condvar/futex.rs
@@ -1,4 +1,5 @@
-use crate::sync::atomic::{AtomicU32, Ordering::Relaxed};
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::Relaxed;
 use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
 use crate::sys::sync::Mutex;
 use crate::time::Duration;
diff --git a/library/std/src/sys/sync/condvar/itron.rs b/library/std/src/sys/sync/condvar/itron.rs
index 3a3039889e9..f6f2b698e49 100644
--- a/library/std/src/sys/sync/condvar/itron.rs
+++ b/library/std/src/sys/sync/condvar/itron.rs
@@ -1,9 +1,13 @@
 //! POSIX conditional variable implementation based on user-space wait queues.
 
-use crate::sys::pal::itron::{
-    abi, error::expect_success_aborting, spin::SpinMutex, task, time::with_tmos_strong,
-};
-use crate::{mem::replace, ptr::NonNull, sys::sync::Mutex, time::Duration};
+use crate::mem::replace;
+use crate::ptr::NonNull;
+use crate::sys::pal::itron::error::expect_success_aborting;
+use crate::sys::pal::itron::spin::SpinMutex;
+use crate::sys::pal::itron::time::with_tmos_strong;
+use crate::sys::pal::itron::{abi, task};
+use crate::sys::sync::Mutex;
+use crate::time::Duration;
 
 // The implementation is inspired by the queue-based implementation shown in
 // Andrew D. Birrell's paper "Implementing Condition Variables with Semaphores"
diff --git a/library/std/src/sys/sync/condvar/pthread.rs b/library/std/src/sys/sync/condvar/pthread.rs
index a2a96410d93..2b4bdfe415c 100644
--- a/library/std/src/sys/sync/condvar/pthread.rs
+++ b/library/std/src/sys/sync/condvar/pthread.rs
@@ -1,6 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
-use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
+use crate::sync::atomic::AtomicPtr;
+use crate::sync::atomic::Ordering::Relaxed;
 use crate::sys::sync::{mutex, Mutex};
 #[cfg(not(target_os = "nto"))]
 use crate::sys::time::TIMESPEC_MAX;
diff --git a/library/std/src/sys/sync/condvar/teeos.rs b/library/std/src/sys/sync/condvar/teeos.rs
index 6457da91c2a..943867cd761 100644
--- a/library/std/src/sys/sync/condvar/teeos.rs
+++ b/library/std/src/sys/sync/condvar/teeos.rs
@@ -1,6 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
-use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
+use crate::sync::atomic::AtomicPtr;
+use crate::sync::atomic::Ordering::Relaxed;
 use crate::sys::sync::mutex::{self, Mutex};
 use crate::sys::time::TIMESPEC_MAX;
 use crate::sys_common::lazy_box::{LazyBox, LazyInit};
diff --git a/library/std/src/sys/sync/condvar/windows7.rs b/library/std/src/sys/sync/condvar/windows7.rs
index 07fa5fdd698..56eeeda551e 100644
--- a/library/std/src/sys/sync/condvar/windows7.rs
+++ b/library/std/src/sys/sync/condvar/windows7.rs
@@ -1,7 +1,6 @@
 use crate::cell::UnsafeCell;
-use crate::sys::c;
-use crate::sys::os;
 use crate::sys::sync::{mutex, Mutex};
+use crate::sys::{c, os};
 use crate::time::Duration;
 
 pub struct Condvar {
diff --git a/library/std/src/sys/sync/condvar/xous.rs b/library/std/src/sys/sync/condvar/xous.rs
index 7b218818ef8..007383479a1 100644
--- a/library/std/src/sys/sync/condvar/xous.rs
+++ b/library/std/src/sys/sync/condvar/xous.rs
@@ -1,8 +1,9 @@
+use core::sync::atomic::{AtomicUsize, Ordering};
+
 use crate::os::xous::ffi::{blocking_scalar, scalar};
 use crate::os::xous::services::{ticktimer_server, TicktimerScalar};
 use crate::sys::sync::Mutex;
 use crate::time::Duration;
-use core::sync::atomic::{AtomicUsize, Ordering};
 
 // The implementation is inspired by Andrew D. Birrell's paper
 // "Implementing Condition Variables with Semaphores"
diff --git a/library/std/src/sys/sync/mutex/fuchsia.rs b/library/std/src/sys/sync/mutex/fuchsia.rs
index 5d89e5a13fd..81a6361a83a 100644
--- a/library/std/src/sys/sync/mutex/fuchsia.rs
+++ b/library/std/src/sys/sync/mutex/fuchsia.rs
@@ -37,10 +37,8 @@
 //!
 //! [mutex in Fuchsia's libsync]: https://cs.opensource.google/fuchsia/fuchsia/+/main:zircon/system/ulib/sync/mutex.c
 
-use crate::sync::atomic::{
-    AtomicU32,
-    Ordering::{Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
 use crate::sys::futex::zircon::{
     zx_futex_wait, zx_futex_wake_single_owner, zx_handle_t, zx_thread_self, ZX_ERR_BAD_HANDLE,
     ZX_ERR_BAD_STATE, ZX_ERR_INVALID_ARGS, ZX_ERR_TIMED_OUT, ZX_ERR_WRONG_TYPE, ZX_OK,
diff --git a/library/std/src/sys/sync/mutex/itron.rs b/library/std/src/sys/sync/mutex/itron.rs
index b29c7e1d034..8440ffdd337 100644
--- a/library/std/src/sys/sync/mutex/itron.rs
+++ b/library/std/src/sys/sync/mutex/itron.rs
@@ -2,18 +2,16 @@
 //! `TA_INHERIT` are available.
 #![forbid(unsafe_op_in_unsafe_fn)]
 
-use crate::sys::pal::itron::{
-    abi,
-    error::{expect_success, expect_success_aborting, fail, ItronError},
-    spin::SpinIdOnceCell,
-};
+use crate::sys::pal::itron::abi;
+use crate::sys::pal::itron::error::{expect_success, expect_success_aborting, fail, ItronError};
+use crate::sys::pal::itron::spin::SpinIdOnceCell;
 
 pub struct Mutex {
     /// The ID of the underlying mutex object
     mtx: SpinIdOnceCell<()>,
 }
 
-/// Create a mutex object. This function never panics.
+/// Creates a mutex object. This function never panics.
 fn new_mtx() -> Result<abi::ID, ItronError> {
     ItronError::err_if_negative(unsafe {
         abi::acre_mtx(&abi::T_CMTX {
@@ -31,7 +29,7 @@ impl Mutex {
         Mutex { mtx: SpinIdOnceCell::new() }
     }
 
-    /// Get the inner mutex's ID, which is lazily created.
+    /// Gets the inner mutex's ID, which is lazily created.
     fn raw(&self) -> abi::ID {
         match self.mtx.get_or_try_init(|| new_mtx().map(|id| (id, ()))) {
             Ok((id, ())) => id,
diff --git a/library/std/src/sys/sync/mutex/xous.rs b/library/std/src/sys/sync/mutex/xous.rs
index 1426e48f8b7..63efa5a0210 100644
--- a/library/std/src/sys/sync/mutex/xous.rs
+++ b/library/std/src/sys/sync/mutex/xous.rs
@@ -1,9 +1,7 @@
 use crate::os::xous::ffi::{blocking_scalar, do_yield};
 use crate::os::xous::services::{ticktimer_server, TicktimerScalar};
-use crate::sync::atomic::{
-    AtomicBool, AtomicUsize,
-    Ordering::{Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use crate::sync::atomic::{AtomicBool, AtomicUsize};
 
 pub struct Mutex {
     /// The "locked" value indicates how many threads are waiting on this
diff --git a/library/std/src/sys/sync/once/futex.rs b/library/std/src/sys/sync/once/futex.rs
index 8a231e65ad1..2c8a054282b 100644
--- a/library/std/src/sys/sync/once/futex.rs
+++ b/library/std/src/sys/sync/once/futex.rs
@@ -1,14 +1,12 @@
 use crate::cell::Cell;
 use crate::sync as public;
-use crate::sync::atomic::{
-    AtomicU32,
-    Ordering::{Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
 use crate::sync::once::ExclusiveState;
 use crate::sys::futex::{futex_wait, futex_wake_all};
 
 // On some platforms, the OS is very nice and handles the waiter queue for us.
-// This means we only need one atomic value with 5 states:
+// This means we only need one atomic value with 4 states:
 
 /// No initialization has run yet, and no thread is currently using the Once.
 const INCOMPLETE: u32 = 0;
@@ -19,16 +17,20 @@ const POISONED: u32 = 1;
 /// Some thread is currently attempting to run initialization. It may succeed,
 /// so all future threads need to wait for it to finish.
 const RUNNING: u32 = 2;
-/// Some thread is currently attempting to run initialization and there are threads
-/// waiting for it to finish.
-const QUEUED: u32 = 3;
 /// Initialization has completed and all future calls should finish immediately.
-const COMPLETE: u32 = 4;
+const COMPLETE: u32 = 3;
 
-// Threads wait by setting the state to QUEUED and calling `futex_wait` on the state
+// An additional bit indicates whether there are waiting threads:
+
+/// May only be set if the state is not COMPLETE.
+const QUEUED: u32 = 4;
+
+// Threads wait by setting the QUEUED bit and calling `futex_wait` on the state
 // variable. When the running thread finishes, it will wake all waiting threads using
 // `futex_wake_all`.
 
+const STATE_MASK: u32 = 0b11;
+
 pub struct OnceState {
     poisoned: bool,
     set_state_to: Cell<u32>,
@@ -47,7 +49,7 @@ impl OnceState {
 }
 
 struct CompletionGuard<'a> {
-    state: &'a AtomicU32,
+    state_and_queued: &'a AtomicU32,
     set_state_on_drop_to: u32,
 }
 
@@ -56,32 +58,32 @@ impl<'a> Drop for CompletionGuard<'a> {
         // Use release ordering to propagate changes to all threads checking
         // up on the Once. `futex_wake_all` does its own synchronization, hence
         // we do not need `AcqRel`.
-        if self.state.swap(self.set_state_on_drop_to, Release) == QUEUED {
-            futex_wake_all(self.state);
+        if self.state_and_queued.swap(self.set_state_on_drop_to, Release) & QUEUED != 0 {
+            futex_wake_all(self.state_and_queued);
         }
     }
 }
 
 pub struct Once {
-    state: AtomicU32,
+    state_and_queued: AtomicU32,
 }
 
 impl Once {
     #[inline]
     pub const fn new() -> Once {
-        Once { state: AtomicU32::new(INCOMPLETE) }
+        Once { state_and_queued: AtomicU32::new(INCOMPLETE) }
     }
 
     #[inline]
     pub fn is_completed(&self) -> bool {
         // Use acquire ordering to make all initialization changes visible to the
         // current thread.
-        self.state.load(Acquire) == COMPLETE
+        self.state_and_queued.load(Acquire) == COMPLETE
     }
 
     #[inline]
     pub(crate) fn state(&mut self) -> ExclusiveState {
-        match *self.state.get_mut() {
+        match *self.state_and_queued.get_mut() {
             INCOMPLETE => ExclusiveState::Incomplete,
             POISONED => ExclusiveState::Poisoned,
             COMPLETE => ExclusiveState::Complete,
@@ -89,31 +91,73 @@ impl Once {
         }
     }
 
-    // This uses FnMut to match the API of the generic implementation. As this
-    // implementation is quite light-weight, it is generic over the closure and
-    // so avoids the cost of dynamic dispatch.
     #[cold]
     #[track_caller]
-    pub fn call(&self, ignore_poisoning: bool, f: &mut impl FnMut(&public::OnceState)) {
-        let mut state = self.state.load(Acquire);
+    pub fn wait(&self, ignore_poisoning: bool) {
+        let mut state_and_queued = self.state_and_queued.load(Acquire);
         loop {
+            let state = state_and_queued & STATE_MASK;
+            let queued = state_and_queued & QUEUED != 0;
             match state {
+                COMPLETE => return,
+                POISONED if !ignore_poisoning => {
+                    // Panic to propagate the poison.
+                    panic!("Once instance has previously been poisoned");
+                }
+                _ => {
+                    // Set the QUEUED bit if it has not already been set.
+                    if !queued {
+                        state_and_queued += QUEUED;
+                        if let Err(new) = self.state_and_queued.compare_exchange_weak(
+                            state,
+                            state_and_queued,
+                            Relaxed,
+                            Acquire,
+                        ) {
+                            state_and_queued = new;
+                            continue;
+                        }
+                    }
+
+                    futex_wait(&self.state_and_queued, state_and_queued, None);
+                    state_and_queued = self.state_and_queued.load(Acquire);
+                }
+            }
+        }
+    }
+
+    #[cold]
+    #[track_caller]
+    pub fn call(&self, ignore_poisoning: bool, f: &mut dyn FnMut(&public::OnceState)) {
+        let mut state_and_queued = self.state_and_queued.load(Acquire);
+        loop {
+            let state = state_and_queued & STATE_MASK;
+            let queued = state_and_queued & QUEUED != 0;
+            match state {
+                COMPLETE => return,
                 POISONED if !ignore_poisoning => {
                     // Panic to propagate the poison.
                     panic!("Once instance has previously been poisoned");
                 }
                 INCOMPLETE | POISONED => {
                     // Try to register the current thread as the one running.
-                    if let Err(new) =
-                        self.state.compare_exchange_weak(state, RUNNING, Acquire, Acquire)
-                    {
-                        state = new;
+                    let next = RUNNING + if queued { QUEUED } else { 0 };
+                    if let Err(new) = self.state_and_queued.compare_exchange_weak(
+                        state_and_queued,
+                        next,
+                        Acquire,
+                        Acquire,
+                    ) {
+                        state_and_queued = new;
                         continue;
                     }
+
                     // `waiter_queue` will manage other waiting threads, and
                     // wake them up on drop.
-                    let mut waiter_queue =
-                        CompletionGuard { state: &self.state, set_state_on_drop_to: POISONED };
+                    let mut waiter_queue = CompletionGuard {
+                        state_and_queued: &self.state_and_queued,
+                        set_state_on_drop_to: POISONED,
+                    };
                     // Run the function, letting it know if we're poisoned or not.
                     let f_state = public::OnceState {
                         inner: OnceState {
@@ -125,21 +169,27 @@ impl Once {
                     waiter_queue.set_state_on_drop_to = f_state.inner.set_state_to.get();
                     return;
                 }
-                RUNNING | QUEUED => {
-                    // Set the state to QUEUED if it is not already.
-                    if state == RUNNING
-                        && let Err(new) =
-                            self.state.compare_exchange_weak(RUNNING, QUEUED, Relaxed, Acquire)
-                    {
-                        state = new;
-                        continue;
+                _ => {
+                    // All other values must be RUNNING.
+                    assert!(state == RUNNING);
+
+                    // Set the QUEUED bit if it is not already set.
+                    if !queued {
+                        state_and_queued += QUEUED;
+                        if let Err(new) = self.state_and_queued.compare_exchange_weak(
+                            state,
+                            state_and_queued,
+                            Relaxed,
+                            Acquire,
+                        ) {
+                            state_and_queued = new;
+                            continue;
+                        }
                     }
 
-                    futex_wait(&self.state, QUEUED, None);
-                    state = self.state.load(Acquire);
+                    futex_wait(&self.state_and_queued, state_and_queued, None);
+                    state_and_queued = self.state_and_queued.load(Acquire);
                 }
-                COMPLETE => return,
-                _ => unreachable!("state is never set to invalid values"),
             }
         }
     }
diff --git a/library/std/src/sys/sync/once/no_threads.rs b/library/std/src/sys/sync/once/no_threads.rs
index 11fde1888ba..12c1d9f5a6c 100644
--- a/library/std/src/sys/sync/once/no_threads.rs
+++ b/library/std/src/sys/sync/once/no_threads.rs
@@ -57,6 +57,12 @@ impl Once {
 
     #[cold]
     #[track_caller]
+    pub fn wait(&self, _ignore_poisoning: bool) {
+        panic!("not implementable on this target");
+    }
+
+    #[cold]
+    #[track_caller]
     pub fn call(&self, ignore_poisoning: bool, f: &mut impl FnMut(&public::OnceState)) {
         let state = self.state.get();
         match state {
diff --git a/library/std/src/sys/sync/once/queue.rs b/library/std/src/sys/sync/once/queue.rs
index 730cdb768bd..86f72c82008 100644
--- a/library/std/src/sys/sync/once/queue.rs
+++ b/library/std/src/sys/sync/once/queue.rs
@@ -56,22 +56,21 @@
 //   allowed, so no need for `SeqCst`.
 
 use crate::cell::Cell;
-use crate::fmt;
-use crate::ptr;
-use crate::sync as public;
-use crate::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
+use crate::sync::atomic::Ordering::{AcqRel, Acquire, Release};
+use crate::sync::atomic::{AtomicBool, AtomicPtr};
 use crate::sync::once::ExclusiveState;
 use crate::thread::{self, Thread};
+use crate::{fmt, ptr, sync as public};
 
-type Masked = ();
+type StateAndQueue = *mut ();
 
 pub struct Once {
-    state_and_queue: AtomicPtr<Masked>,
+    state_and_queue: AtomicPtr<()>,
 }
 
 pub struct OnceState {
     poisoned: bool,
-    set_state_on_drop_to: Cell<*mut Masked>,
+    set_state_on_drop_to: Cell<StateAndQueue>,
 }
 
 // Four states that a Once can be in, encoded into the lower bits of
@@ -83,7 +82,8 @@ const COMPLETE: usize = 0x3;
 
 // Mask to learn about the state. All other bits are the queue of waiters if
 // this is in the RUNNING state.
-const STATE_MASK: usize = 0x3;
+const STATE_MASK: usize = 0b11;
+const QUEUE_MASK: usize = !STATE_MASK;
 
 // Representation of a node in the linked list of waiters, used while in the
 // RUNNING state.
@@ -95,15 +95,23 @@ const STATE_MASK: usize = 0x3;
 struct Waiter {
     thread: Cell<Option<Thread>>,
     signaled: AtomicBool,
-    next: *const Waiter,
+    next: Cell<*const Waiter>,
 }
 
 // Head of a linked list of waiters.
 // Every node is a struct on the stack of a waiting thread.
 // Will wake up the waiters when it gets dropped, i.e. also on panic.
 struct WaiterQueue<'a> {
-    state_and_queue: &'a AtomicPtr<Masked>,
-    set_state_on_drop_to: *mut Masked,
+    state_and_queue: &'a AtomicPtr<()>,
+    set_state_on_drop_to: StateAndQueue,
+}
+
+fn to_queue(current: StateAndQueue) -> *const Waiter {
+    current.mask(QUEUE_MASK).cast()
+}
+
+fn to_state(current: StateAndQueue) -> usize {
+    current.addr() & STATE_MASK
 }
 
 impl Once {
@@ -119,7 +127,7 @@ impl Once {
         // operations visible to us, and, this being a fast path, weaker
         // ordering helps with performance. This `Acquire` synchronizes with
         // `Release` operations on the slow path.
-        self.state_and_queue.load(Ordering::Acquire).addr() == COMPLETE
+        self.state_and_queue.load(Acquire).addr() == COMPLETE
     }
 
     #[inline]
@@ -132,6 +140,25 @@ impl Once {
         }
     }
 
+    #[cold]
+    #[track_caller]
+    pub fn wait(&self, ignore_poisoning: bool) {
+        let mut current = self.state_and_queue.load(Acquire);
+        loop {
+            let state = to_state(current);
+            match state {
+                COMPLETE => return,
+                POISONED if !ignore_poisoning => {
+                    // Panic to propagate the poison.
+                    panic!("Once instance has previously been poisoned");
+                }
+                _ => {
+                    current = wait(&self.state_and_queue, current, !ignore_poisoning);
+                }
+            }
+        }
+    }
+
     // This is a non-generic function to reduce the monomorphization cost of
     // using `call_once` (this isn't exactly a trivial or small implementation).
     //
@@ -146,9 +173,10 @@ impl Once {
     #[cold]
     #[track_caller]
     pub fn call(&self, ignore_poisoning: bool, init: &mut dyn FnMut(&public::OnceState)) {
-        let mut state_and_queue = self.state_and_queue.load(Ordering::Acquire);
+        let mut current = self.state_and_queue.load(Acquire);
         loop {
-            match state_and_queue.addr() {
+            let state = to_state(current);
+            match state {
                 COMPLETE => break,
                 POISONED if !ignore_poisoning => {
                     // Panic to propagate the poison.
@@ -156,16 +184,16 @@ impl Once {
                 }
                 POISONED | INCOMPLETE => {
                     // Try to register this thread as the one RUNNING.
-                    let exchange_result = self.state_and_queue.compare_exchange(
-                        state_and_queue,
-                        ptr::without_provenance_mut(RUNNING),
-                        Ordering::Acquire,
-                        Ordering::Acquire,
-                    );
-                    if let Err(old) = exchange_result {
-                        state_and_queue = old;
+                    if let Err(new) = self.state_and_queue.compare_exchange_weak(
+                        current,
+                        current.mask(QUEUE_MASK).wrapping_byte_add(RUNNING),
+                        Acquire,
+                        Acquire,
+                    ) {
+                        current = new;
                         continue;
                     }
+
                     // `waiter_queue` will manage other waiting threads, and
                     // wake them up on drop.
                     let mut waiter_queue = WaiterQueue {
@@ -176,54 +204,57 @@ impl Once {
                     // poisoned or not.
                     let init_state = public::OnceState {
                         inner: OnceState {
-                            poisoned: state_and_queue.addr() == POISONED,
+                            poisoned: state == POISONED,
                             set_state_on_drop_to: Cell::new(ptr::without_provenance_mut(COMPLETE)),
                         },
                     };
                     init(&init_state);
                     waiter_queue.set_state_on_drop_to = init_state.inner.set_state_on_drop_to.get();
-                    break;
+                    return;
                 }
                 _ => {
                     // All other values must be RUNNING with possibly a
                     // pointer to the waiter queue in the more significant bits.
-                    assert!(state_and_queue.addr() & STATE_MASK == RUNNING);
-                    wait(&self.state_and_queue, state_and_queue);
-                    state_and_queue = self.state_and_queue.load(Ordering::Acquire);
+                    assert!(state == RUNNING);
+                    current = wait(&self.state_and_queue, current, true);
                 }
             }
         }
     }
 }
 
-fn wait(state_and_queue: &AtomicPtr<Masked>, mut current_state: *mut Masked) {
-    // Note: the following code was carefully written to avoid creating a
-    // mutable reference to `node` that gets aliased.
+fn wait(
+    state_and_queue: &AtomicPtr<()>,
+    mut current: StateAndQueue,
+    return_on_poisoned: bool,
+) -> StateAndQueue {
+    let node = &Waiter {
+        thread: Cell::new(Some(thread::current())),
+        signaled: AtomicBool::new(false),
+        next: Cell::new(ptr::null()),
+    };
+
     loop {
-        // Don't queue this thread if the status is no longer running,
-        // otherwise we will not be woken up.
-        if current_state.addr() & STATE_MASK != RUNNING {
-            return;
+        let state = to_state(current);
+        let queue = to_queue(current);
+
+        // If initialization has finished, return.
+        if state == COMPLETE || (return_on_poisoned && state == POISONED) {
+            return current;
         }
 
-        // Create the node for our current thread.
-        let node = Waiter {
-            thread: Cell::new(Some(thread::current())),
-            signaled: AtomicBool::new(false),
-            next: current_state.with_addr(current_state.addr() & !STATE_MASK) as *const Waiter,
-        };
-        let me = core::ptr::addr_of!(node) as *const Masked as *mut Masked;
+        // Update the node for our current thread.
+        node.next.set(queue);
 
         // Try to slide in the node at the head of the linked list, making sure
         // that another thread didn't just replace the head of the linked list.
-        let exchange_result = state_and_queue.compare_exchange(
-            current_state,
-            me.with_addr(me.addr() | RUNNING),
-            Ordering::Release,
-            Ordering::Relaxed,
-        );
-        if let Err(old) = exchange_result {
-            current_state = old;
+        if let Err(new) = state_and_queue.compare_exchange_weak(
+            current,
+            ptr::from_ref(node).wrapping_byte_add(state) as StateAndQueue,
+            Release,
+            Acquire,
+        ) {
+            current = new;
             continue;
         }
 
@@ -232,14 +263,15 @@ fn wait(state_and_queue: &AtomicPtr<Masked>, mut current_state: *mut Masked) {
         // would drop our `Waiter` node and leave a hole in the linked list
         // (and a dangling reference). Guard against spurious wakeups by
         // reparking ourselves until we are signaled.
-        while !node.signaled.load(Ordering::Acquire) {
+        while !node.signaled.load(Acquire) {
             // If the managing thread happens to signal and unpark us before we
             // can park ourselves, the result could be this thread never gets
             // unparked. Luckily `park` comes with the guarantee that if it got
             // an `unpark` just before on an unparked thread it does not park.
             thread::park();
         }
-        break;
+
+        return state_and_queue.load(Acquire);
     }
 }
 
@@ -253,11 +285,10 @@ impl fmt::Debug for Once {
 impl Drop for WaiterQueue<'_> {
     fn drop(&mut self) {
         // Swap out our state with however we finished.
-        let state_and_queue =
-            self.state_and_queue.swap(self.set_state_on_drop_to, Ordering::AcqRel);
+        let current = self.state_and_queue.swap(self.set_state_on_drop_to, AcqRel);
 
         // We should only ever see an old state which was RUNNING.
-        assert_eq!(state_and_queue.addr() & STATE_MASK, RUNNING);
+        assert_eq!(current.addr() & STATE_MASK, RUNNING);
 
         // Walk the entire linked list of waiters and wake them up (in lifo
         // order, last to register is first to wake up).
@@ -266,16 +297,13 @@ impl Drop for WaiterQueue<'_> {
             // free `node` if there happens to be has a spurious wakeup.
             // So we have to take out the `thread` field and copy the pointer to
             // `next` first.
-            let mut queue =
-                state_and_queue.with_addr(state_and_queue.addr() & !STATE_MASK) as *const Waiter;
+            let mut queue = to_queue(current);
             while !queue.is_null() {
-                let next = (*queue).next;
+                let next = (*queue).next.get();
                 let thread = (*queue).thread.take().unwrap();
-                (*queue).signaled.store(true, Ordering::Release);
-                // ^- FIXME (maybe): This is another case of issue #55005
-                // `store()` has a potentially dangling ref to `signaled`.
-                queue = next;
+                (*queue).signaled.store(true, Release);
                 thread.unpark();
+                queue = next;
             }
         }
     }
diff --git a/library/std/src/sys/sync/rwlock/futex.rs b/library/std/src/sys/sync/rwlock/futex.rs
index aa0de900238..75ecc2ab5c5 100644
--- a/library/std/src/sys/sync/rwlock/futex.rs
+++ b/library/std/src/sys/sync/rwlock/futex.rs
@@ -1,7 +1,5 @@
-use crate::sync::atomic::{
-    AtomicU32,
-    Ordering::{Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
 use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
 
 pub struct RwLock {
@@ -222,7 +220,7 @@ impl RwLock {
         }
     }
 
-    /// Wake up waiting threads after unlocking.
+    /// Wakes up waiting threads after unlocking.
     ///
     /// If both are waiting, this will wake up only one writer, but will fall
     /// back to waking up readers if there was no writer to wake up.
diff --git a/library/std/src/sys/sync/rwlock/queue.rs b/library/std/src/sys/sync/rwlock/queue.rs
index 337cc6c2ca0..458c16516bb 100644
--- a/library/std/src/sys/sync/rwlock/queue.rs
+++ b/library/std/src/sys/sync/rwlock/queue.rs
@@ -111,10 +111,8 @@ use crate::cell::OnceCell;
 use crate::hint::spin_loop;
 use crate::mem;
 use crate::ptr::{self, null_mut, without_provenance_mut, NonNull};
-use crate::sync::atomic::{
-    AtomicBool, AtomicPtr,
-    Ordering::{AcqRel, Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release};
+use crate::sync::atomic::{AtomicBool, AtomicPtr};
 use crate::thread::{self, Thread};
 
 // Locking uses exponential backoff. `SPIN_COUNT` indicates how many times the
@@ -186,7 +184,7 @@ struct Node {
 }
 
 impl Node {
-    /// Create a new queue node.
+    /// Creates a new queue node.
     fn new(write: bool) -> Node {
         Node {
             next: AtomicLink::new(None),
diff --git a/library/std/src/sys/sync/rwlock/solid.rs b/library/std/src/sys/sync/rwlock/solid.rs
index a8fef685ceb..05371402020 100644
--- a/library/std/src/sys/sync/rwlock/solid.rs
+++ b/library/std/src/sys/sync/rwlock/solid.rs
@@ -1,13 +1,9 @@
 //! A readers-writer lock implementation backed by the SOLID kernel extension.
 #![forbid(unsafe_op_in_unsafe_fn)]
 
-use crate::sys::pal::{
-    abi,
-    itron::{
-        error::{expect_success, expect_success_aborting, fail, ItronError},
-        spin::SpinIdOnceCell,
-    },
-};
+use crate::sys::pal::abi;
+use crate::sys::pal::itron::error::{expect_success, expect_success_aborting, fail, ItronError};
+use crate::sys::pal::itron::spin::SpinIdOnceCell;
 
 pub struct RwLock {
     /// The ID of the underlying mutex object
@@ -28,7 +24,7 @@ impl RwLock {
         RwLock { rwl: SpinIdOnceCell::new() }
     }
 
-    /// Get the inner mutex's ID, which is lazily created.
+    /// Gets the inner mutex's ID, which is lazily created.
     fn raw(&self) -> abi::ID {
         match self.rwl.get_or_try_init(|| new_rwl().map(|id| (id, ()))) {
             Ok((id, ())) => id,
diff --git a/library/std/src/sys/sync/thread_parking/darwin.rs b/library/std/src/sys/sync/thread_parking/darwin.rs
index 973c08f0317..96e3d23c332 100644
--- a/library/std/src/sys/sync/thread_parking/darwin.rs
+++ b/library/std/src/sys/sync/thread_parking/darwin.rs
@@ -13,10 +13,8 @@
 #![allow(non_camel_case_types)]
 
 use crate::pin::Pin;
-use crate::sync::atomic::{
-    AtomicI8,
-    Ordering::{Acquire, Release},
-};
+use crate::sync::atomic::AtomicI8;
+use crate::sync::atomic::Ordering::{Acquire, Release};
 use crate::time::Duration;
 
 type dispatch_semaphore_t = *mut crate::ffi::c_void;
diff --git a/library/std/src/sys/sync/thread_parking/futex.rs b/library/std/src/sys/sync/thread_parking/futex.rs
index 034eececb2a..ce852eaadc4 100644
--- a/library/std/src/sys/sync/thread_parking/futex.rs
+++ b/library/std/src/sys/sync/thread_parking/futex.rs
@@ -36,7 +36,7 @@ pub struct Parker {
 // Ordering::Release when writing NOTIFIED (the 'token') in unpark(), and using
 // Ordering::Acquire when checking for this state in park().
 impl Parker {
-    /// Construct the futex parker. The UNIX parker implementation
+    /// Constructs the futex parker. The UNIX parker implementation
     /// requires this to happen in-place.
     pub unsafe fn new_in_place(parker: *mut Parker) {
         unsafe { parker.write(Self { state: Atomic::new(EMPTY) }) };
diff --git a/library/std/src/sys/sync/thread_parking/id.rs b/library/std/src/sys/sync/thread_parking/id.rs
index 04667439660..a7b07b509df 100644
--- a/library/std/src/sys/sync/thread_parking/id.rs
+++ b/library/std/src/sys/sync/thread_parking/id.rs
@@ -9,10 +9,8 @@
 
 use crate::cell::UnsafeCell;
 use crate::pin::Pin;
-use crate::sync::atomic::{
-    fence, AtomicI8,
-    Ordering::{Acquire, Relaxed, Release},
-};
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use crate::sync::atomic::{fence, AtomicI8};
 use crate::sys::thread_parking::{current, park, park_timeout, unpark, ThreadId};
 use crate::time::Duration;
 
@@ -30,7 +28,7 @@ impl Parker {
         Parker { state: AtomicI8::new(EMPTY), tid: UnsafeCell::new(None) }
     }
 
-    /// Create a new thread parker. UNIX requires this to happen in-place.
+    /// Creates a new thread parker. UNIX requires this to happen in-place.
     pub unsafe fn new_in_place(parker: *mut Parker) {
         parker.write(Parker::new())
     }
diff --git a/library/std/src/sys/sync/thread_parking/pthread.rs b/library/std/src/sys/sync/thread_parking/pthread.rs
index fdac1096dbf..c64600e9e29 100644
--- a/library/std/src/sys/sync/thread_parking/pthread.rs
+++ b/library/std/src/sys/sync/thread_parking/pthread.rs
@@ -92,7 +92,7 @@ pub struct Parker {
 }
 
 impl Parker {
-    /// Construct the UNIX parker in-place.
+    /// Constructs the UNIX parker in-place.
     ///
     /// # Safety
     /// The constructed parker must never be moved.
diff --git a/library/std/src/sys/sync/thread_parking/windows7.rs b/library/std/src/sys/sync/thread_parking/windows7.rs
index 3a8d40dc5cf..1000b63b6d0 100644
--- a/library/std/src/sys/sync/thread_parking/windows7.rs
+++ b/library/std/src/sys/sync/thread_parking/windows7.rs
@@ -57,14 +57,13 @@
 // [3]: https://docs.microsoft.com/en-us/archive/msdn-magazine/2012/november/windows-with-c-the-evolution-of-synchronization-in-windows-and-c
 // [4]: Windows Internals, Part 1, ISBN 9780735671300
 
+use core::ffi::c_void;
+
 use crate::pin::Pin;
-use crate::sync::atomic::{
-    AtomicI8,
-    Ordering::{Acquire, Release},
-};
+use crate::sync::atomic::AtomicI8;
+use crate::sync::atomic::Ordering::{Acquire, Release};
 use crate::sys::{c, dur2timeout};
 use crate::time::Duration;
-use core::ffi::c_void;
 
 pub struct Parker {
     state: AtomicI8,
@@ -95,7 +94,7 @@ const NOTIFIED: i8 = 1;
 // Ordering::Release when writing NOTIFIED (the 'token') in unpark(), and using
 // Ordering::Acquire when reading this state in park() after waking up.
 impl Parker {
-    /// Construct the Windows parker. The UNIX parker implementation
+    /// Constructs the Windows parker. The UNIX parker implementation
     /// requires this to happen in-place.
     pub unsafe fn new_in_place(parker: *mut Parker) {
         parker.write(Self { state: AtomicI8::new(EMPTY) });
@@ -185,16 +184,15 @@ impl Parker {
 
 #[cfg(target_vendor = "win7")]
 mod keyed_events {
-    use super::{Parker, EMPTY, NOTIFIED};
-    use crate::sys::c;
     use core::pin::Pin;
     use core::ptr;
-    use core::sync::atomic::{
-        AtomicPtr,
-        Ordering::{Acquire, Relaxed},
-    };
+    use core::sync::atomic::AtomicPtr;
+    use core::sync::atomic::Ordering::{Acquire, Relaxed};
     use core::time::Duration;
 
+    use super::{Parker, EMPTY, NOTIFIED};
+    use crate::sys::c;
+
     pub unsafe fn park(parker: Pin<&Parker>) {
         // Wait for unpark() to produce this event.
         c::NtWaitForKeyedEvent(keyed_event_handle(), parker.ptr(), 0, ptr::null_mut());
diff --git a/library/std/src/sys/sync/thread_parking/xous.rs b/library/std/src/sys/sync/thread_parking/xous.rs
index 0bd0462d77d..64b6f731f23 100644
--- a/library/std/src/sys/sync/thread_parking/xous.rs
+++ b/library/std/src/sys/sync/thread_parking/xous.rs
@@ -2,10 +2,8 @@ use crate::os::xous::ffi::{blocking_scalar, scalar};
 use crate::os::xous::services::{ticktimer_server, TicktimerScalar};
 use crate::pin::Pin;
 use crate::ptr;
-use crate::sync::atomic::{
-    AtomicI8,
-    Ordering::{Acquire, Release},
-};
+use crate::sync::atomic::AtomicI8;
+use crate::sync::atomic::Ordering::{Acquire, Release};
 use crate::time::Duration;
 
 const NOTIFIED: i8 = 1;