diff options
Diffstat (limited to 'library/std/src/sys')
64 files changed, 1309 insertions, 1252 deletions
diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index bbd1d840e92..8f70cefc601 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -9,8 +9,6 @@ pub mod cmath; pub mod os_str; pub mod path; pub mod sync; -#[allow(dead_code)] -#[allow(unused_imports)] pub mod thread_local; // FIXME(117276): remove this, move feature implementations into individual diff --git a/library/std/src/sys/os_str/bytes.rs b/library/std/src/sys/os_str/bytes.rs index 4ca3f1cd185..18b969bca85 100644 --- a/library/std/src/sys/os_str/bytes.rs +++ b/library/std/src/sys/os_str/bytes.rs @@ -11,8 +11,6 @@ use crate::str; use crate::sync::Arc; use crate::sys_common::{AsInner, IntoInner}; -use core::str::Utf8Chunks; - #[cfg(test)] mod tests; @@ -29,7 +27,7 @@ pub struct Slice { impl fmt::Debug for Slice { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&Utf8Chunks::new(&self.inner).debug(), f) + fmt::Debug::fmt(&self.inner.utf8_chunks().debug(), f) } } @@ -41,7 +39,7 @@ impl fmt::Display for Slice { return "".fmt(f); } - for chunk in Utf8Chunks::new(&self.inner) { + for chunk in self.inner.utf8_chunks() { let valid = chunk.valid(); // If we successfully decoded the whole chunk as a valid string then // we can return a direct formatting of the string which will also @@ -198,6 +196,12 @@ impl Buf { pub fn into_rc(&self) -> Rc<Slice> { self.as_slice().into_rc() } + + /// Part of a hack to make PathBuf::push/pop more efficient. + #[inline] + pub(crate) fn as_mut_vec_for_path_buf(&mut self) -> &mut Vec<u8> { + &mut self.inner + } } impl Slice { diff --git a/library/std/src/sys/os_str/wtf8.rs b/library/std/src/sys/os_str/wtf8.rs index 352bd735903..b3ceb55802d 100644 --- a/library/std/src/sys/os_str/wtf8.rs +++ b/library/std/src/sys/os_str/wtf8.rs @@ -158,6 +158,12 @@ impl Buf { pub fn into_rc(&self) -> Rc<Slice> { self.as_slice().into_rc() } + + /// Part of a hack to make PathBuf::push/pop more efficient. + #[inline] + pub(crate) fn as_mut_vec_for_path_buf(&mut self) -> &mut Vec<u8> { + self.inner.as_mut_vec_for_path_buf() + } } impl Slice { diff --git a/library/std/src/sys/pal/common/alloc.rs b/library/std/src/sys/pal/common/alloc.rs index 8cf9ef68047..598b6db71f5 100644 --- a/library/std/src/sys/pal/common/alloc.rs +++ b/library/std/src/sys/pal/common/alloc.rs @@ -23,6 +23,7 @@ pub const MIN_ALIGN: usize = 8; #[cfg(any( target_arch = "x86_64", target_arch = "aarch64", + target_arch = "arm64ec", target_arch = "loongarch64", target_arch = "mips64", target_arch = "mips64r6", diff --git a/library/std/src/sys/pal/hermit/alloc.rs b/library/std/src/sys/pal/hermit/alloc.rs index de550987a43..2cd0db90940 100644 --- a/library/std/src/sys/pal/hermit/alloc.rs +++ b/library/std/src/sys/pal/hermit/alloc.rs @@ -1,4 +1,4 @@ -use super::abi; +use super::hermit_abi; use crate::alloc::{GlobalAlloc, Layout, System}; use crate::ptr; @@ -6,11 +6,11 @@ use crate::ptr; unsafe impl GlobalAlloc for System { #[inline] unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - abi::malloc(layout.size(), layout.align()) + hermit_abi::malloc(layout.size(), layout.align()) } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - let addr = abi::malloc(layout.size(), layout.align()); + let addr = hermit_abi::malloc(layout.size(), layout.align()); if !addr.is_null() { ptr::write_bytes(addr, 0x00, layout.size()); @@ -21,11 +21,11 @@ unsafe impl GlobalAlloc for System { #[inline] unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - abi::free(ptr, layout.size(), layout.align()) + hermit_abi::free(ptr, layout.size(), layout.align()) } #[inline] unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - abi::realloc(ptr, layout.size(), layout.align(), new_size) + hermit_abi::realloc(ptr, layout.size(), layout.align(), new_size) } } diff --git a/library/std/src/sys/pal/hermit/fd.rs b/library/std/src/sys/pal/hermit/fd.rs index 962577bb1ed..d7dab08cfbd 100644 --- a/library/std/src/sys/pal/hermit/fd.rs +++ b/library/std/src/sys/pal/hermit/fd.rs @@ -1,6 +1,6 @@ #![unstable(reason = "not public", issue = "none", feature = "fd")] -use super::abi; +use super::hermit_abi; use crate::io::{self, Read}; use crate::os::hermit::io::{FromRawFd, OwnedFd, RawFd}; use crate::sys::cvt; @@ -16,7 +16,8 @@ pub struct FileDesc { impl FileDesc { pub fn read(&self, buf: &mut [u8]) -> io::Result<usize> { - let result = cvt(unsafe { abi::read(self.fd.as_raw_fd(), buf.as_mut_ptr(), buf.len()) })?; + let result = + cvt(unsafe { hermit_abi::read(self.fd.as_raw_fd(), buf.as_mut_ptr(), buf.len()) })?; Ok(result as usize) } @@ -26,7 +27,8 @@ impl FileDesc { } pub fn write(&self, buf: &[u8]) -> io::Result<usize> { - let result = cvt(unsafe { abi::write(self.fd.as_raw_fd(), buf.as_ptr(), buf.len()) })?; + let result = + cvt(unsafe { hermit_abi::write(self.fd.as_raw_fd(), buf.as_ptr(), buf.len()) })?; Ok(result as usize) } @@ -49,8 +51,8 @@ impl FileDesc { unsupported() } - pub fn fstat(&self, stat: *mut abi::stat) -> io::Result<()> { - cvt(unsafe { abi::fstat(self.fd.as_raw_fd(), stat) })?; + pub fn fstat(&self, stat: *mut hermit_abi::stat) -> io::Result<()> { + cvt(unsafe { hermit_abi::fstat(self.fd.as_raw_fd(), stat) })?; Ok(()) } } diff --git a/library/std/src/sys/pal/hermit/fs.rs b/library/std/src/sys/pal/hermit/fs.rs index 6519cc22f1f..a4a16e6e86b 100644 --- a/library/std/src/sys/pal/hermit/fs.rs +++ b/library/std/src/sys/pal/hermit/fs.rs @@ -1,8 +1,8 @@ -use super::abi::{ +use super::fd::FileDesc; +use super::hermit_abi::{ self, dirent64, stat as stat_struct, DT_DIR, DT_LNK, DT_REG, DT_UNKNOWN, O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY, S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, }; -use super::fd::FileDesc; use crate::ffi::{CStr, OsStr, OsString}; use crate::fmt; use crate::io::{self, Error, ErrorKind}; @@ -47,7 +47,7 @@ impl InnerReadDir { pub struct ReadDir { inner: Arc<InnerReadDir>, - pos: i64, + pos: usize, } impl ReadDir { @@ -197,38 +197,31 @@ impl Iterator for ReadDir { fn next(&mut self) -> Option<io::Result<DirEntry>> { let mut counter: usize = 0; - let mut offset: i64 = 0; + let mut offset: usize = 0; // loop over all directory entries and search the entry for the current position loop { // leave function, if the loop reaches the of the buffer (with all entries) - if offset >= self.inner.dir.len().try_into().unwrap() { + if offset >= self.inner.dir.len() { return None; } - let dir = unsafe { - &*(self.inner.dir.as_ptr().offset(offset.try_into().unwrap()) as *const dirent64) - }; + let dir = unsafe { &*(self.inner.dir.as_ptr().add(offset) as *const dirent64) }; - if counter == self.pos.try_into().unwrap() { + if counter == self.pos { self.pos += 1; // After dirent64, the file name is stored. d_reclen represents the length of the dirent64 // plus the length of the file name. Consequently, file name has a size of d_reclen minus // the size of dirent64. The file name is always a C string and terminated by `\0`. // Consequently, we are able to ignore the last byte. - let name_bytes = unsafe { - core::slice::from_raw_parts( - &dir.d_name as *const _ as *const u8, - dir.d_reclen as usize - core::mem::size_of::<dirent64>() - 1, - ) - .to_vec() - }; + let name_bytes = + unsafe { CStr::from_ptr(&dir.d_name as *const _ as *const i8).to_bytes() }; let entry = DirEntry { root: self.inner.root.clone(), ino: dir.d_ino, type_: dir.d_type as u32, - name: OsString::from_vec(name_bytes), + name: OsString::from_vec(name_bytes.to_vec()), }; return Some(Ok(entry)); @@ -237,7 +230,7 @@ impl Iterator for ReadDir { counter += 1; // move to the next dirent64, which is directly stored after the previous one - offset = offset + dir.d_off; + offset = offset + usize::from(dir.d_reclen); } } } @@ -365,7 +358,7 @@ impl File { mode = 0; } - let fd = unsafe { cvt(abi::open(path.as_ptr(), flags, mode))? }; + let fd = unsafe { cvt(hermit_abi::open(path.as_ptr(), flags, mode))? }; Ok(File(unsafe { FileDesc::from_raw_fd(fd as i32) })) } @@ -446,7 +439,7 @@ impl DirBuilder { pub fn mkdir(&self, path: &Path) -> io::Result<()> { run_path_with_cstr(path, &|path| { - cvt(unsafe { abi::mkdir(path.as_ptr(), self.mode) }).map(|_| ()) + cvt(unsafe { hermit_abi::mkdir(path.as_ptr(), self.mode) }).map(|_| ()) }) } @@ -508,7 +501,8 @@ impl FromRawFd for File { } pub fn readdir(path: &Path) -> io::Result<ReadDir> { - let fd_raw = run_path_with_cstr(path, &|path| cvt(unsafe { abi::opendir(path.as_ptr()) }))?; + let fd_raw = + run_path_with_cstr(path, &|path| cvt(unsafe { hermit_abi::opendir(path.as_ptr()) }))?; let fd = unsafe { FileDesc::from_raw_fd(fd_raw as i32) }; let root = path.to_path_buf(); @@ -519,8 +513,9 @@ pub fn readdir(path: &Path) -> io::Result<ReadDir> { // reserve memory to receive all directory entries vec.resize(sz, 0); - let readlen = - unsafe { abi::getdents64(fd.as_raw_fd(), vec.as_mut_ptr() as *mut dirent64, sz) }; + let readlen = unsafe { + hermit_abi::getdents64(fd.as_raw_fd(), vec.as_mut_ptr() as *mut dirent64, sz) + }; if readlen > 0 { // shrink down to the minimal size vec.resize(readlen.try_into().unwrap(), 0); @@ -529,7 +524,7 @@ pub fn readdir(path: &Path) -> io::Result<ReadDir> { // if the buffer is too small, getdents64 returns EINVAL // otherwise, getdents64 returns an error number - if readlen != (-abi::errno::EINVAL).into() { + if readlen != (-hermit_abi::errno::EINVAL).into() { return Err(Error::from_raw_os_error(readlen.try_into().unwrap())); } @@ -547,7 +542,7 @@ pub fn readdir(path: &Path) -> io::Result<ReadDir> { } pub fn unlink(path: &Path) -> io::Result<()> { - run_path_with_cstr(path, &|path| cvt(unsafe { abi::unlink(path.as_ptr()) }).map(|_| ())) + run_path_with_cstr(path, &|path| cvt(unsafe { hermit_abi::unlink(path.as_ptr()) }).map(|_| ())) } pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { @@ -559,7 +554,7 @@ pub fn set_perm(_p: &Path, _perm: FilePermissions) -> io::Result<()> { } pub fn rmdir(path: &Path) -> io::Result<()> { - run_path_with_cstr(path, &|path| cvt(unsafe { abi::rmdir(path.as_ptr()) }).map(|_| ())) + run_path_with_cstr(path, &|path| cvt(unsafe { hermit_abi::rmdir(path.as_ptr()) }).map(|_| ())) } pub fn remove_dir_all(_path: &Path) -> io::Result<()> { @@ -581,7 +576,7 @@ pub fn link(_original: &Path, _link: &Path) -> io::Result<()> { pub fn stat(path: &Path) -> io::Result<FileAttr> { run_path_with_cstr(path, &|path| { let mut stat_val: stat_struct = unsafe { mem::zeroed() }; - cvt(unsafe { abi::stat(path.as_ptr(), &mut stat_val) })?; + cvt(unsafe { hermit_abi::stat(path.as_ptr(), &mut stat_val) })?; Ok(FileAttr::from_stat(stat_val)) }) } @@ -589,7 +584,7 @@ pub fn stat(path: &Path) -> io::Result<FileAttr> { pub fn lstat(path: &Path) -> io::Result<FileAttr> { run_path_with_cstr(path, &|path| { let mut stat_val: stat_struct = unsafe { mem::zeroed() }; - cvt(unsafe { abi::lstat(path.as_ptr(), &mut stat_val) })?; + cvt(unsafe { hermit_abi::lstat(path.as_ptr(), &mut stat_val) })?; Ok(FileAttr::from_stat(stat_val)) }) } diff --git a/library/std/src/sys/pal/hermit/futex.rs b/library/std/src/sys/pal/hermit/futex.rs index 427d8ff6f2e..571b2885658 100644 --- a/library/std/src/sys/pal/hermit/futex.rs +++ b/library/std/src/sys/pal/hermit/futex.rs @@ -1,4 +1,4 @@ -use super::abi; +use super::hermit_abi; use crate::ptr::null; use crate::sync::atomic::AtomicU32; use crate::time::Duration; @@ -8,32 +8,32 @@ pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) - // // Overflows are rounded up to an infinite timeout (None). let timespec = timeout.and_then(|dur| { - Some(abi::timespec { + Some(hermit_abi::timespec { tv_sec: dur.as_secs().try_into().ok()?, tv_nsec: dur.subsec_nanos().into(), }) }); let r = unsafe { - abi::futex_wait( + hermit_abi::futex_wait( futex.as_ptr(), expected, - timespec.as_ref().map_or(null(), |t| t as *const abi::timespec), - abi::FUTEX_RELATIVE_TIMEOUT, + timespec.as_ref().map_or(null(), |t| t as *const hermit_abi::timespec), + hermit_abi::FUTEX_RELATIVE_TIMEOUT, ) }; - r != -abi::errno::ETIMEDOUT + r != -hermit_abi::errno::ETIMEDOUT } #[inline] pub fn futex_wake(futex: &AtomicU32) -> bool { - unsafe { abi::futex_wake(futex.as_ptr(), 1) > 0 } + unsafe { hermit_abi::futex_wake(futex.as_ptr(), 1) > 0 } } #[inline] pub fn futex_wake_all(futex: &AtomicU32) { unsafe { - abi::futex_wake(futex.as_ptr(), i32::MAX); + hermit_abi::futex_wake(futex.as_ptr(), i32::MAX); } } diff --git a/library/std/src/sys/pal/hermit/mod.rs b/library/std/src/sys/pal/hermit/mod.rs index 910935541bd..a64323a3a29 100644 --- a/library/std/src/sys/pal/hermit/mod.rs +++ b/library/std/src/sys/pal/hermit/mod.rs @@ -39,7 +39,7 @@ pub mod thread_local_key; pub mod time; use crate::io::ErrorKind; -use crate::os::hermit::abi; +use crate::os::hermit::hermit_abi; pub fn unsupported<T>() -> crate::io::Result<T> { Err(unsupported_err()) @@ -54,7 +54,7 @@ pub fn unsupported_err() -> crate::io::Error { pub fn abort_internal() -> ! { unsafe { - abi::abort(); + hermit_abi::abort(); } } @@ -62,7 +62,7 @@ pub fn hashmap_random_keys() -> (u64, u64) { let mut buf = [0; 16]; let mut slice = &mut buf[..]; while !slice.is_empty() { - let res = cvt(unsafe { abi::read_entropy(slice.as_mut_ptr(), slice.len(), 0) }) + let res = cvt(unsafe { hermit_abi::read_entropy(slice.as_mut_ptr(), slice.len(), 0) }) .expect("failed to generate random hashmap keys"); slice = &mut slice[res as usize..]; } @@ -109,31 +109,31 @@ pub unsafe extern "C" fn runtime_entry( let result = main(argc as isize, argv); run_dtors(); - abi::exit(result); + hermit_abi::exit(result); } #[inline] pub(crate) fn is_interrupted(errno: i32) -> bool { - errno == abi::errno::EINTR + errno == hermit_abi::errno::EINTR } pub fn decode_error_kind(errno: i32) -> ErrorKind { match errno { - abi::errno::EACCES => ErrorKind::PermissionDenied, - abi::errno::EADDRINUSE => ErrorKind::AddrInUse, - abi::errno::EADDRNOTAVAIL => ErrorKind::AddrNotAvailable, - abi::errno::EAGAIN => ErrorKind::WouldBlock, - abi::errno::ECONNABORTED => ErrorKind::ConnectionAborted, - abi::errno::ECONNREFUSED => ErrorKind::ConnectionRefused, - abi::errno::ECONNRESET => ErrorKind::ConnectionReset, - abi::errno::EEXIST => ErrorKind::AlreadyExists, - abi::errno::EINTR => ErrorKind::Interrupted, - abi::errno::EINVAL => ErrorKind::InvalidInput, - abi::errno::ENOENT => ErrorKind::NotFound, - abi::errno::ENOTCONN => ErrorKind::NotConnected, - abi::errno::EPERM => ErrorKind::PermissionDenied, - abi::errno::EPIPE => ErrorKind::BrokenPipe, - abi::errno::ETIMEDOUT => ErrorKind::TimedOut, + hermit_abi::errno::EACCES => ErrorKind::PermissionDenied, + hermit_abi::errno::EADDRINUSE => ErrorKind::AddrInUse, + hermit_abi::errno::EADDRNOTAVAIL => ErrorKind::AddrNotAvailable, + hermit_abi::errno::EAGAIN => ErrorKind::WouldBlock, + hermit_abi::errno::ECONNABORTED => ErrorKind::ConnectionAborted, + hermit_abi::errno::ECONNREFUSED => ErrorKind::ConnectionRefused, + hermit_abi::errno::ECONNRESET => ErrorKind::ConnectionReset, + hermit_abi::errno::EEXIST => ErrorKind::AlreadyExists, + hermit_abi::errno::EINTR => ErrorKind::Interrupted, + hermit_abi::errno::EINVAL => ErrorKind::InvalidInput, + hermit_abi::errno::ENOENT => ErrorKind::NotFound, + hermit_abi::errno::ENOTCONN => ErrorKind::NotConnected, + hermit_abi::errno::EPERM => ErrorKind::PermissionDenied, + hermit_abi::errno::EPIPE => ErrorKind::BrokenPipe, + hermit_abi::errno::ETIMEDOUT => ErrorKind::TimedOut, _ => ErrorKind::Uncategorized, } } diff --git a/library/std/src/sys/pal/hermit/net.rs b/library/std/src/sys/pal/hermit/net.rs index 23ac71cb9f2..00dbca86a4b 100644 --- a/library/std/src/sys/pal/hermit/net.rs +++ b/library/std/src/sys/pal/hermit/net.rs @@ -175,23 +175,12 @@ impl Socket { } pub fn read_vectored(&self, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> { - let mut size: isize = 0; - - for i in bufs.iter_mut() { - let ret: isize = - cvt(unsafe { netc::read(self.0.as_raw_fd(), i.as_mut_ptr(), i.len()) })?; - - if ret != 0 { - size += ret; - } - } - - Ok(size.try_into().unwrap()) + crate::io::default_read_vectored(|b| self.read(b), bufs) } #[inline] pub fn is_read_vectored(&self) -> bool { - true + false } fn recv_from_with_flags(&self, buf: &mut [u8], flags: i32) -> io::Result<(usize, SocketAddr)> { @@ -225,17 +214,11 @@ impl Socket { } pub fn write_vectored(&self, bufs: &[IoSlice<'_>]) -> io::Result<usize> { - let mut size: isize = 0; - - for i in bufs.iter() { - size += cvt(unsafe { netc::write(self.0.as_raw_fd(), i.as_ptr(), i.len()) })?; - } - - Ok(size.try_into().unwrap()) + crate::io::default_write_vectored(|b| self.write(b), bufs) } pub fn is_write_vectored(&self) -> bool { - true + false } pub fn set_timeout(&self, dur: Option<Duration>, kind: i32) -> io::Result<()> { diff --git a/library/std/src/sys/pal/hermit/os.rs b/library/std/src/sys/pal/hermit/os.rs index 1f9affbf773..cc678123831 100644 --- a/library/std/src/sys/pal/hermit/os.rs +++ b/library/std/src/sys/pal/hermit/os.rs @@ -1,4 +1,4 @@ -use super::abi; +use super::hermit_abi; use crate::collections::HashMap; use crate::error::Error as StdError; use crate::ffi::{CStr, OsStr, OsString}; @@ -14,11 +14,11 @@ use crate::vec; use core::slice::memchr; pub fn errno() -> i32 { - unsafe { abi::get_errno() } + unsafe { hermit_abi::get_errno() } } pub fn error_string(errno: i32) -> String { - abi::error_string(errno).to_string() + hermit_abi::error_string(errno).to_string() } pub fn getcwd() -> io::Result<PathBuf> { @@ -197,10 +197,10 @@ pub fn home_dir() -> Option<PathBuf> { pub fn exit(code: i32) -> ! { unsafe { - abi::exit(code); + hermit_abi::exit(code); } } pub fn getpid() -> u32 { - unsafe { abi::getpid() } + unsafe { hermit_abi::getpid() } } diff --git a/library/std/src/sys/pal/hermit/stdio.rs b/library/std/src/sys/pal/hermit/stdio.rs index ac54385e8ce..777c57b391c 100644 --- a/library/std/src/sys/pal/hermit/stdio.rs +++ b/library/std/src/sys/pal/hermit/stdio.rs @@ -1,4 +1,4 @@ -use super::abi; +use super::hermit_abi; use crate::io; use crate::io::{IoSlice, IoSliceMut}; @@ -37,7 +37,7 @@ impl io::Write for Stdout { fn write(&mut self, data: &[u8]) -> io::Result<usize> { let len; - unsafe { len = abi::write(1, data.as_ptr() as *const u8, data.len()) } + unsafe { len = hermit_abi::write(1, data.as_ptr() as *const u8, data.len()) } if len < 0 { Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stdout is not able to print")) @@ -49,7 +49,7 @@ impl io::Write for Stdout { fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result<usize> { let len; - unsafe { len = abi::write(1, data.as_ptr() as *const u8, data.len()) } + unsafe { len = hermit_abi::write(1, data.as_ptr() as *const u8, data.len()) } if len < 0 { Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stdout is not able to print")) @@ -78,7 +78,7 @@ impl io::Write for Stderr { fn write(&mut self, data: &[u8]) -> io::Result<usize> { let len; - unsafe { len = abi::write(2, data.as_ptr() as *const u8, data.len()) } + unsafe { len = hermit_abi::write(2, data.as_ptr() as *const u8, data.len()) } if len < 0 { Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stderr is not able to print")) @@ -90,7 +90,7 @@ impl io::Write for Stderr { fn write_vectored(&mut self, data: &[IoSlice<'_>]) -> io::Result<usize> { let len; - unsafe { len = abi::write(2, data.as_ptr() as *const u8, data.len()) } + unsafe { len = hermit_abi::write(2, data.as_ptr() as *const u8, data.len()) } if len < 0 { Err(io::const_io_error!(io::ErrorKind::Uncategorized, "Stderr is not able to print")) diff --git a/library/std/src/sys/pal/hermit/thread.rs b/library/std/src/sys/pal/hermit/thread.rs index 4fe6b12a95b..b336dcd6860 100644 --- a/library/std/src/sys/pal/hermit/thread.rs +++ b/library/std/src/sys/pal/hermit/thread.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use super::abi; +use super::hermit_abi; use super::thread_local_dtor::run_dtors; use crate::ffi::CStr; use crate::io; @@ -9,7 +9,7 @@ use crate::num::NonZero; use crate::ptr; use crate::time::Duration; -pub type Tid = abi::Tid; +pub type Tid = hermit_abi::Tid; pub struct Thread { tid: Tid, @@ -27,10 +27,10 @@ impl Thread { core_id: isize, ) -> io::Result<Thread> { let p = Box::into_raw(Box::new(p)); - let tid = abi::spawn2( + let tid = hermit_abi::spawn2( thread_start, p.expose_provenance(), - abi::Priority::into(abi::NORMAL_PRIO), + hermit_abi::Priority::into(hermit_abi::NORMAL_PRIO), stack, core_id, ); @@ -62,7 +62,7 @@ impl Thread { #[inline] pub fn yield_now() { unsafe { - abi::yield_now(); + hermit_abi::yield_now(); } } @@ -74,13 +74,13 @@ impl Thread { #[inline] pub fn sleep(dur: Duration) { unsafe { - abi::usleep(dur.as_micros() as u64); + hermit_abi::usleep(dur.as_micros() as u64); } } pub fn join(self) { unsafe { - let _ = abi::join(self.tid); + let _ = hermit_abi::join(self.tid); } } @@ -98,5 +98,5 @@ impl Thread { } pub fn available_parallelism() -> io::Result<NonZero<usize>> { - unsafe { Ok(NonZero::new_unchecked(abi::get_processor_count())) } + unsafe { Ok(NonZero::new_unchecked(hermit_abi::get_processor_count())) } } diff --git a/library/std/src/sys/pal/hermit/time.rs b/library/std/src/sys/pal/hermit/time.rs index d6f9e4c1476..2bf24462fa8 100644 --- a/library/std/src/sys/pal/hermit/time.rs +++ b/library/std/src/sys/pal/hermit/time.rs @@ -1,8 +1,6 @@ #![allow(dead_code)] -use super::abi; -use super::abi::timespec; -use super::abi::{CLOCK_MONOTONIC, CLOCK_REALTIME, NSEC_PER_SEC}; +use super::hermit_abi::{self, timespec, CLOCK_MONOTONIC, CLOCK_REALTIME, NSEC_PER_SEC}; use crate::cmp::Ordering; use crate::ops::{Add, AddAssign, Sub, SubAssign}; use crate::time::Duration; @@ -106,7 +104,8 @@ pub struct Instant(Timespec); impl Instant { pub fn now() -> Instant { let mut time: Timespec = Timespec::zero(); - let _ = unsafe { abi::clock_gettime(CLOCK_MONOTONIC, core::ptr::addr_of_mut!(time.t)) }; + let _ = + unsafe { hermit_abi::clock_gettime(CLOCK_MONOTONIC, core::ptr::addr_of_mut!(time.t)) }; Instant(time) } @@ -207,7 +206,8 @@ impl SystemTime { pub fn now() -> SystemTime { let mut time: Timespec = Timespec::zero(); - let _ = unsafe { abi::clock_gettime(CLOCK_REALTIME, core::ptr::addr_of_mut!(time.t)) }; + let _ = + unsafe { hermit_abi::clock_gettime(CLOCK_REALTIME, core::ptr::addr_of_mut!(time.t)) }; SystemTime(time) } diff --git a/library/std/src/sys/pal/sgx/thread.rs b/library/std/src/sys/pal/sgx/thread.rs index e2df57b1a1f..7d271e6d2b6 100644 --- a/library/std/src/sys/pal/sgx/thread.rs +++ b/library/std/src/sys/pal/sgx/thread.rs @@ -67,7 +67,7 @@ mod task_queue { pub mod wait_notify { use crate::pin::Pin; use crate::sync::Arc; - use crate::sys_common::thread_parking::Parker; + use crate::sys::sync::Parker; pub struct Notifier(Arc<Parker>); diff --git a/library/std/src/sys/pal/teeos/mod.rs b/library/std/src/sys/pal/teeos/mod.rs index c392a0ea264..6dd465a12ed 100644 --- a/library/std/src/sys/pal/teeos/mod.rs +++ b/library/std/src/sys/pal/teeos/mod.rs @@ -30,8 +30,6 @@ pub mod thread; pub mod thread_local_dtor; #[path = "../unix/thread_local_key.rs"] pub mod thread_local_key; -#[path = "../unsupported/thread_parking.rs"] -pub mod thread_parking; #[allow(non_upper_case_globals)] #[path = "../unix/time.rs"] pub mod time; diff --git a/library/std/src/sys/pal/uefi/mod.rs b/library/std/src/sys/pal/uefi/mod.rs index 562b00c2c01..48b74df1384 100644 --- a/library/std/src/sys/pal/uefi/mod.rs +++ b/library/std/src/sys/pal/uefi/mod.rs @@ -30,8 +30,6 @@ pub mod stdio; pub mod thread; #[path = "../unsupported/thread_local_key.rs"] pub mod thread_local_key; -#[path = "../unsupported/thread_parking.rs"] -pub mod thread_parking; pub mod time; mod helpers; diff --git a/library/std/src/sys/pal/unix/alloc.rs b/library/std/src/sys/pal/unix/alloc.rs index af0089978ec..eb3a57c212b 100644 --- a/library/std/src/sys/pal/unix/alloc.rs +++ b/library/std/src/sys/pal/unix/alloc.rs @@ -13,7 +13,13 @@ unsafe impl GlobalAlloc for System { if layout.align() <= MIN_ALIGN && layout.align() <= layout.size() { libc::malloc(layout.size()) as *mut u8 } else { - #[cfg(target_os = "macos")] + // `posix_memalign` returns a non-aligned value if supplied a very + // large alignment on older versions of Apple's platforms (unknown + // exactly which version range, but the issue is definitely + // present in macOS 10.14 and iOS 13.3). + // + // <https://github.com/rust-lang/rust/issues/30170> + #[cfg(target_vendor = "apple")] { if layout.align() > (1 << 31) { return ptr::null_mut(); @@ -53,51 +59,27 @@ unsafe impl GlobalAlloc for System { } cfg_if::cfg_if! { + // We use posix_memalign wherever possible, but some targets have very incomplete POSIX coverage + // so we need a fallback for those. if #[cfg(any( - target_os = "android", - target_os = "illumos", - target_os = "redox", - target_os = "solaris", - target_os = "espidf", target_os = "horizon", target_os = "vita", ))] { #[inline] unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { - // On android we currently target API level 9 which unfortunately - // doesn't have the `posix_memalign` API used below. Instead we use - // `memalign`, but this unfortunately has the property on some systems - // where the memory returned cannot be deallocated by `free`! - // - // Upon closer inspection, however, this appears to work just fine with - // Android, so for this platform we should be fine to call `memalign` - // (which is present in API level 9). Some helpful references could - // possibly be chromium using memalign [1], attempts at documenting that - // memalign + free is ok [2] [3], or the current source of chromium - // which still uses memalign on android [4]. - // - // [1]: https://codereview.chromium.org/10796020/ - // [2]: https://code.google.com/p/android/issues/detail?id=35391 - // [3]: https://bugs.chromium.org/p/chromium/issues/detail?id=138579 - // [4]: https://chromium.googlesource.com/chromium/src/base/+/master/ - // /memory/aligned_memory.cc libc::memalign(layout.align(), layout.size()) as *mut u8 } - } else if #[cfg(target_os = "wasi")] { - #[inline] - unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { - // C11 aligned_alloc requires that the size be a multiple of the alignment. - // Layout already checks that the size rounded up doesn't overflow isize::MAX. - let align = layout.align(); - let size = layout.size().next_multiple_of(align); - libc::aligned_alloc(align, size) as *mut u8 - } } else { #[inline] unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { let mut out = ptr::null_mut(); - // posix_memalign requires that the alignment be a multiple of `sizeof(void*)`. - // Since these are all powers of 2, we can just use max. + // We prefer posix_memalign over aligned_alloc since it is more widely available, and + // since with aligned_alloc, implementations are making almost arbitrary choices for + // which alignments are "supported", making it hard to use. For instance, some + // implementations require the size to be a multiple of the alignment (wasi emmalloc), + // while others require the alignment to be at least the pointer size (Illumos, macOS). + // posix_memalign only has one, clear requirement: that the alignment be a multiple of + // `sizeof(void*)`. Since these are all powers of 2, we can just use max. let align = layout.align().max(crate::mem::size_of::<usize>()); let ret = libc::posix_memalign(&mut out, align, layout.size()); if ret != 0 { ptr::null_mut() } else { out as *mut u8 } diff --git a/library/std/src/sys/pal/unix/args.rs b/library/std/src/sys/pal/unix/args.rs index acf8100d47f..db2ec73148e 100644 --- a/library/std/src/sys/pal/unix/args.rs +++ b/library/std/src/sys/pal/unix/args.rs @@ -5,8 +5,9 @@ #![allow(dead_code)] // runtime init functions not used during testing -use crate::ffi::OsString; +use crate::ffi::{CStr, OsString}; use crate::fmt; +use crate::os::unix::ffi::OsStringExt; use crate::vec; /// One-time global initialization. @@ -16,7 +17,46 @@ pub unsafe fn init(argc: isize, argv: *const *const u8) { /// Returns the command line arguments pub fn args() -> Args { - imp::args() + let (argc, argv) = imp::argc_argv(); + + let mut vec = Vec::with_capacity(argc as usize); + + for i in 0..argc { + // SAFETY: `argv` is non-null if `argc` is positive, and it is + // guaranteed to be at least as long as `argc`, so reading from it + // should be safe. + let ptr = unsafe { argv.offset(i).read() }; + + // Some C commandline parsers (e.g. GLib and Qt) are replacing already + // handled arguments in `argv` with `NULL` and move them to the end. + // + // Since they can't directly ensure updates to `argc` as well, this + // means that `argc` might be bigger than the actual number of + // non-`NULL` pointers in `argv` at this point. + // + // To handle this we simply stop iterating at the first `NULL` + // argument. `argv` is also guaranteed to be `NULL`-terminated so any + // non-`NULL` arguments after the first `NULL` can safely be ignored. + if ptr.is_null() { + // NOTE: On Apple platforms, `-[NSProcessInfo arguments]` does not + // stop iterating here, but instead `continue`, always iterating + // up until it reached `argc`. + // + // This difference will only matter in very specific circumstances + // where `argc`/`argv` have been modified, but in unexpected ways, + // so it likely doesn't really matter which option we choose. + // See the following PR for further discussion: + // <https://github.com/rust-lang/rust/pull/125225> + break; + } + + // SAFETY: Just checked that the pointer is not NULL, and arguments + // are otherwise guaranteed to be valid C strings. + let cstr = unsafe { CStr::from_ptr(ptr) }; + vec.push(OsStringExt::from_vec(cstr.to_bytes().to_vec())); + } + + Args { iter: vec.into_iter() } } pub struct Args { @@ -75,9 +115,7 @@ impl DoubleEndedIterator for Args { target_os = "hurd", ))] mod imp { - use super::Args; - use crate::ffi::{CStr, OsString}; - use crate::os::unix::prelude::*; + use crate::ffi::c_char; use crate::ptr; use crate::sync::atomic::{AtomicIsize, AtomicPtr, Ordering}; @@ -98,12 +136,10 @@ mod imp { } #[inline(always)] - pub unsafe fn init(_argc: isize, _argv: *const *const u8) { - // On Linux-GNU, we rely on `ARGV_INIT_ARRAY` below to initialize - // `ARGC` and `ARGV`. But in Miri that does not actually happen so we - // still initialize here. - #[cfg(any(miri, not(all(target_os = "linux", target_env = "gnu"))))] - really_init(_argc, _argv); + pub unsafe fn init(argc: isize, argv: *const *const u8) { + // on GNU/Linux if we are main then we will init argv and argc twice, it "duplicates work" + // BUT edge-cases are real: only using .init_array can break most emulators, dlopen, etc. + really_init(argc, argv); } /// glibc passes argc, argv, and envp to functions in .init_array, as a non-standard extension. @@ -128,173 +164,78 @@ mod imp { init_wrapper }; - pub fn args() -> Args { - Args { iter: clone().into_iter() } - } - - fn clone() -> Vec<OsString> { - unsafe { - // Load ARGC and ARGV, which hold the unmodified system-provided - // argc/argv, so we can read the pointed-to memory without atomics - // or synchronization. - // - // If either ARGC or ARGV is still zero or null, then either there - // really are no arguments, or someone is asking for `args()` - // before initialization has completed, and we return an empty - // list. - let argv = ARGV.load(Ordering::Relaxed); - let argc = if argv.is_null() { 0 } else { ARGC.load(Ordering::Relaxed) }; - let mut args = Vec::with_capacity(argc as usize); - for i in 0..argc { - let ptr = *argv.offset(i) as *const libc::c_char; - - // Some C commandline parsers (e.g. GLib and Qt) are replacing already - // handled arguments in `argv` with `NULL` and move them to the end. That - // means that `argc` might be bigger than the actual number of non-`NULL` - // pointers in `argv` at this point. - // - // To handle this we simply stop iterating at the first `NULL` argument. - // - // `argv` is also guaranteed to be `NULL`-terminated so any non-`NULL` arguments - // after the first `NULL` can safely be ignored. - if ptr.is_null() { - break; - } - - let cstr = CStr::from_ptr(ptr); - args.push(OsStringExt::from_vec(cstr.to_bytes().to_vec())); - } - - args - } + pub fn argc_argv() -> (isize, *const *const c_char) { + // Load ARGC and ARGV, which hold the unmodified system-provided + // argc/argv, so we can read the pointed-to memory without atomics or + // synchronization. + // + // If either ARGC or ARGV is still zero or null, then either there + // really are no arguments, or someone is asking for `args()` before + // initialization has completed, and we return an empty list. + let argv = ARGV.load(Ordering::Relaxed); + let argc = if argv.is_null() { 0 } else { ARGC.load(Ordering::Relaxed) }; + + // Cast from `*mut *const u8` to `*const *const c_char` + (argc, argv.cast()) } } -#[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" -))] +// Use `_NSGetArgc` and `_NSGetArgv` on Apple platforms. +// +// Even though these have underscores in their names, they've been available +// since since the first versions of both macOS and iOS, and are declared in +// the header `crt_externs.h`. +// +// NOTE: This header was added to the iOS 13.0 SDK, which has been the source +// of a great deal of confusion in the past about the availability of these +// APIs. +// +// NOTE(madsmtm): This has not strictly been verified to not cause App Store +// rejections; if this is found to be the case, the previous implementation +// of this used `[[NSProcessInfo processInfo] arguments]`. +#[cfg(target_vendor = "apple")] mod imp { - use super::Args; - use crate::ffi::CStr; - - pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} + use crate::ffi::{c_char, c_int}; - #[cfg(target_os = "macos")] - pub fn args() -> Args { - use crate::os::unix::prelude::*; - extern "C" { - // These functions are in crt_externs.h. - fn _NSGetArgc() -> *mut libc::c_int; - fn _NSGetArgv() -> *mut *mut *mut libc::c_char; - } - - let vec = unsafe { - let (argc, argv) = - (*_NSGetArgc() as isize, *_NSGetArgv() as *const *const libc::c_char); - (0..argc as isize) - .map(|i| { - let bytes = CStr::from_ptr(*argv.offset(i)).to_bytes().to_vec(); - OsStringExt::from_vec(bytes) - }) - .collect::<Vec<_>>() - }; - Args { iter: vec.into_iter() } + pub unsafe fn init(_argc: isize, _argv: *const *const u8) { + // No need to initialize anything in here, `libdyld.dylib` has already + // done the work for us. } - // As _NSGetArgc and _NSGetArgv aren't mentioned in iOS docs - // and use underscores in their names - they're most probably - // are considered private and therefore should be avoided. - // Here is another way to get arguments using the Objective-C - // runtime. - // - // In general it looks like: - // res = Vec::new() - // let args = [[NSProcessInfo processInfo] arguments] - // for i in (0..[args count]) - // res.push([args objectAtIndex:i]) - // res - #[cfg(any( - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos" - ))] - pub fn args() -> Args { - use crate::ffi::{c_char, c_void, OsString}; - use crate::mem; - use crate::str; - - type Sel = *const c_void; - type NsId = *const c_void; - type NSUInteger = usize; - + pub fn argc_argv() -> (isize, *const *const c_char) { extern "C" { - fn sel_registerName(name: *const c_char) -> Sel; - fn objc_getClass(class_name: *const c_char) -> NsId; - - // This must be transmuted to an appropriate function pointer type before being called. - fn objc_msgSend(); - } - - const MSG_SEND_PTR: unsafe extern "C" fn() = objc_msgSend; - const MSG_SEND_NO_ARGUMENTS_RETURN_PTR: unsafe extern "C" fn(NsId, Sel) -> *const c_void = - unsafe { mem::transmute(MSG_SEND_PTR) }; - const MSG_SEND_NO_ARGUMENTS_RETURN_NSUINTEGER: unsafe extern "C" fn( - NsId, - Sel, - ) -> NSUInteger = unsafe { mem::transmute(MSG_SEND_PTR) }; - const MSG_SEND_NSINTEGER_ARGUMENT_RETURN_PTR: unsafe extern "C" fn( - NsId, - Sel, - NSUInteger, - ) - -> *const c_void = unsafe { mem::transmute(MSG_SEND_PTR) }; - - let mut res = Vec::new(); - - unsafe { - let process_info_sel = sel_registerName(c"processInfo".as_ptr()); - let arguments_sel = sel_registerName(c"arguments".as_ptr()); - let count_sel = sel_registerName(c"count".as_ptr()); - let object_at_index_sel = sel_registerName(c"objectAtIndex:".as_ptr()); - let utf8string_sel = sel_registerName(c"UTF8String".as_ptr()); - - let klass = objc_getClass(c"NSProcessInfo".as_ptr()); - // `+[NSProcessInfo processInfo]` returns an object with +0 retain count, so no need to manually `retain/release`. - let info = MSG_SEND_NO_ARGUMENTS_RETURN_PTR(klass, process_info_sel); - - // `-[NSProcessInfo arguments]` returns an object with +0 retain count, so no need to manually `retain/release`. - let args = MSG_SEND_NO_ARGUMENTS_RETURN_PTR(info, arguments_sel); - - let cnt = MSG_SEND_NO_ARGUMENTS_RETURN_NSUINTEGER(args, count_sel); - for i in 0..cnt { - // `-[NSArray objectAtIndex:]` returns an object whose lifetime is tied to the array, so no need to manually `retain/release`. - let ns_string = - MSG_SEND_NSINTEGER_ARGUMENT_RETURN_PTR(args, object_at_index_sel, i); - // The lifetime of this pointer is tied to the NSString, as well as the current autorelease pool, which is why we heap-allocate the string below. - let utf_c_str: *const c_char = - MSG_SEND_NO_ARGUMENTS_RETURN_PTR(ns_string, utf8string_sel).cast(); - let bytes = CStr::from_ptr(utf_c_str).to_bytes(); - res.push(OsString::from(str::from_utf8(bytes).unwrap())) - } + // These functions are in crt_externs.h. + fn _NSGetArgc() -> *mut c_int; + fn _NSGetArgv() -> *mut *mut *mut c_char; } - Args { iter: res.into_iter() } + // SAFETY: The returned pointer points to a static initialized early + // in the program lifetime by `libdyld.dylib`, and as such is always + // valid. + // + // NOTE: Similar to `_NSGetEnviron`, there technically isn't anything + // protecting us against concurrent modifications to this, and there + // doesn't exist a lock that we can take. Instead, it is generally + // expected that it's only modified in `main` / before other code + // runs, so reading this here should be fine. + let argc = unsafe { _NSGetArgc().read() }; + // SAFETY: Same as above. + let argv = unsafe { _NSGetArgv().read() }; + + // Cast from `*mut *mut c_char` to `*const *const c_char` + (argc as isize, argv.cast()) } } #[cfg(any(target_os = "espidf", target_os = "vita"))] mod imp { - use super::Args; + use crate::ffi::c_char; + use crate::ptr; #[inline(always)] pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - pub fn args() -> Args { - Args { iter: Vec::new().into_iter() } + pub fn argc_argv() -> (isize, *const *const c_char) { + (0, ptr::null()) } } diff --git a/library/std/src/sys/pal/unix/fd.rs b/library/std/src/sys/pal/unix/fd.rs index 48e83b04ef4..1701717db59 100644 --- a/library/std/src/sys/pal/unix/fd.rs +++ b/library/std/src/sys/pal/unix/fd.rs @@ -33,25 +33,22 @@ pub struct FileDesc(OwnedFd); // with the man page quoting that if the count of bytes to read is // greater than `SSIZE_MAX` the result is "unspecified". // -// On macOS, however, apparently the 64-bit libc is either buggy or +// On Apple targets however, apparently the 64-bit libc is either buggy or // intentionally showing odd behavior by rejecting any read with a size // larger than or equal to INT_MAX. To handle both of these the read // size is capped on both platforms. -#[cfg(target_os = "macos")] -const READ_LIMIT: usize = libc::c_int::MAX as usize - 1; -#[cfg(not(target_os = "macos"))] -const READ_LIMIT: usize = libc::ssize_t::MAX as usize; +const READ_LIMIT: usize = if cfg!(target_vendor = "apple") { + libc::c_int::MAX as usize - 1 +} else { + libc::ssize_t::MAX as usize +}; #[cfg(any( target_os = "dragonfly", target_os = "freebsd", - target_os = "ios", - target_os = "tvos", - target_os = "macos", target_os = "netbsd", target_os = "openbsd", - target_os = "watchos", - target_os = "visionos", + target_vendor = "apple", ))] const fn max_iov() -> usize { libc::IOV_MAX as usize @@ -72,17 +69,13 @@ const fn max_iov() -> usize { target_os = "dragonfly", target_os = "emscripten", target_os = "freebsd", - target_os = "ios", - target_os = "tvos", target_os = "linux", - target_os = "macos", target_os = "netbsd", target_os = "nto", target_os = "openbsd", target_os = "horizon", target_os = "vita", - target_os = "watchos", - target_os = "visionos", + target_vendor = "apple", )))] const fn max_iov() -> usize { 16 // The minimum value required by POSIX. @@ -201,13 +194,10 @@ impl FileDesc { target_os = "fuchsia", target_os = "hurd", target_os = "illumos", - target_os = "ios", - target_os = "tvos", target_os = "linux", - target_os = "macos", target_os = "netbsd", target_os = "openbsd", - target_os = "watchos", + target_vendor = "apple", )))] pub fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result<usize> { io::default_read_vectored(|b| self.read_at(b, offset), bufs) @@ -241,15 +231,7 @@ impl FileDesc { Ok(ret as usize) } - // We support old MacOS and iOS versions that do not have `preadv`. There is - // no `syscall` possible in these platform. - #[cfg(any( - all(target_os = "android", target_pointer_width = "32"), - target_os = "ios", // ios 14.0 - target_os = "tvos", // tvos 14.0 - target_os = "macos", // macos 11.0 - target_os = "watchos", // watchos 7.0 - ))] + #[cfg(all(target_os = "android", target_pointer_width = "32"))] pub fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result<usize> { super::weak::weak!(fn preadv64(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); @@ -269,6 +251,35 @@ impl FileDesc { } } + // We support old MacOS, iOS, watchOS, tvOS and visionOS. `preadv` was added in the following + // Apple OS versions: + // ios 14.0 + // tvos 14.0 + // macos 11.0 + // watchos 7.0 + // + // These versions may be newer than the minimum supported versions of OS's we support so we must + // use "weak" linking. + #[cfg(target_vendor = "apple")] + pub fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result<usize> { + super::weak::weak!(fn preadv(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); + + match preadv.get() { + Some(preadv) => { + let ret = cvt(unsafe { + preadv( + self.as_raw_fd(), + bufs.as_mut_ptr() as *mut libc::iovec as *const libc::iovec, + cmp::min(bufs.len(), max_iov()) as libc::c_int, + offset as _, + ) + })?; + Ok(ret as usize) + } + None => io::default_read_vectored(|b| self.read_at(b, offset), bufs), + } + } + pub fn write(&self, buf: &[u8]) -> io::Result<usize> { let ret = cvt(unsafe { libc::write( @@ -360,13 +371,10 @@ impl FileDesc { target_os = "fuchsia", target_os = "hurd", target_os = "illumos", - target_os = "ios", - target_os = "tvos", target_os = "linux", - target_os = "macos", target_os = "netbsd", target_os = "openbsd", - target_os = "watchos", + target_vendor = "apple", )))] pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result<usize> { io::default_write_vectored(|b| self.write_at(b, offset), bufs) @@ -400,15 +408,7 @@ impl FileDesc { Ok(ret as usize) } - // We support old MacOS and iOS versions that do not have `pwritev`. There is - // no `syscall` possible in these platform. - #[cfg(any( - all(target_os = "android", target_pointer_width = "32"), - target_os = "ios", // ios 14.0 - target_os = "tvos", // tvos 14.0 - target_os = "macos", // macos 11.0 - target_os = "watchos", // watchos 7.0 - ))] + #[cfg(all(target_os = "android", target_pointer_width = "32"))] pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result<usize> { super::weak::weak!(fn pwritev64(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); @@ -428,6 +428,35 @@ impl FileDesc { } } + // We support old MacOS, iOS, watchOS, tvOS and visionOS. `pwritev` was added in the following + // Apple OS versions: + // ios 14.0 + // tvos 14.0 + // macos 11.0 + // watchos 7.0 + // + // These versions may be newer than the minimum supported versions of OS's we support so we must + // use "weak" linking. + #[cfg(target_vendor = "apple")] + pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result<usize> { + super::weak::weak!(fn pwritev(libc::c_int, *const libc::iovec, libc::c_int, off64_t) -> isize); + + match pwritev.get() { + Some(pwritev) => { + let ret = cvt(unsafe { + pwritev( + self.as_raw_fd(), + bufs.as_ptr() as *const libc::iovec, + cmp::min(bufs.len(), max_iov()) as libc::c_int, + offset as _, + ) + })?; + Ok(ret as usize) + } + None => io::default_write_vectored(|b| self.write_at(b, offset), bufs), + } + } + #[cfg(not(any( target_env = "newlib", target_os = "solaris", diff --git a/library/std/src/sys/pal/unix/fs.rs b/library/std/src/sys/pal/unix/fs.rs index 3456155509e..fbbd40bfb79 100644 --- a/library/std/src/sys/pal/unix/fs.rs +++ b/library/std/src/sys/pal/unix/fs.rs @@ -20,14 +20,7 @@ use crate::sys::time::SystemTime; use crate::sys::{cvt, cvt_r}; use crate::sys_common::{AsInner, AsInnerMut, FromInner, IntoInner}; -#[cfg(any( - all(target_os = "linux", target_env = "gnu"), - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", -))] +#[cfg(any(all(target_os = "linux", target_env = "gnu"), target_vendor = "apple"))] use crate::sys::weak::syscall; #[cfg(any(target_os = "android", target_os = "macos", target_os = "solaris"))] use crate::sys::weak::weak; @@ -35,13 +28,9 @@ use crate::sys::weak::weak; use libc::{c_int, mode_t}; #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "solaris", - all(target_os = "linux", target_env = "gnu") + all(target_os = "linux", target_env = "gnu"), + target_vendor = "apple", ))] use libc::c_char; #[cfg(any( @@ -198,20 +187,16 @@ cfg_has_statx! {{ return Some(Err(err)); } - // `ENOSYS` might come from a faulty FUSE driver. - // - // Other errors are not a good enough indicator either -- it is - // known that `EPERM` can be returned as a result of using seccomp to - // block the syscall. + // We're not yet entirely sure whether `statx` is usable on this kernel + // or not. Syscalls can return errors from things other than the kernel + // per se, e.g. `EPERM` can be returned if seccomp is used to block the + // syscall, or `ENOSYS` might be returned from a faulty FUSE driver. // // Availability is checked by performing a call which expects `EFAULT` // if the syscall is usable. // // See: https://github.com/rust-lang/rust/issues/65662 // - // FIXME this can probably just do the call if `EPERM` was received, but - // previous iteration of the code checked it for all errors and for now - // this is retained. // FIXME what about transient conditions like `ENOMEM`? let err2 = cvt(statx(0, ptr::null(), 0, libc::STATX_ALL, ptr::null_mut())) .err() @@ -379,13 +364,7 @@ pub struct FilePermissions { pub struct FileTimes { accessed: Option<SystemTime>, modified: Option<SystemTime>, - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" - ))] + #[cfg(target_vendor = "apple")] created: Option<SystemTime>, } @@ -558,15 +537,7 @@ impl FileAttr { SystemTime::new(self.stat.st_atim.tv_sec as i64, self.stat.st_atim.tv_nsec as i64) } - #[cfg(any( - target_os = "freebsd", - target_os = "openbsd", - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - ))] + #[cfg(any(target_os = "freebsd", target_os = "openbsd", target_vendor = "apple"))] pub fn created(&self) -> io::Result<SystemTime> { SystemTime::new(self.stat.st_birthtime as i64, self.stat.st_birthtime_nsec as i64) } @@ -574,12 +545,8 @@ impl FileAttr { #[cfg(not(any( target_os = "freebsd", target_os = "openbsd", - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "vita", + target_vendor = "apple", )))] pub fn created(&self) -> io::Result<SystemTime> { cfg_has_statx! { @@ -659,13 +626,7 @@ impl FileTimes { self.modified = Some(t); } - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" - ))] + #[cfg(target_vendor = "apple")] pub fn set_created(&mut self, t: SystemTime) { self.created = Some(t); } @@ -868,8 +829,43 @@ impl Iterator for ReadDir { } } +/// Aborts the process if a file desceriptor is not open, if debug asserts are enabled +/// +/// Many IO syscalls can't be fully trusted about EBADF error codes because those +/// might get bubbled up from a remote FUSE server rather than the file descriptor +/// in the current process being invalid. +/// +/// So we check file flags instead which live on the file descriptor and not the underlying file. +/// The downside is that it costs an extra syscall, so we only do it for debug. +#[inline] +pub(crate) fn debug_assert_fd_is_open(fd: RawFd) { + use crate::sys::os::errno; + + // this is similar to assert_unsafe_precondition!() but it doesn't require const + if core::ub_checks::check_library_ub() { + if unsafe { libc::fcntl(fd, libc::F_GETFD) } == -1 && errno() == libc::EBADF { + rtabort!("IO Safety violation: owned file descriptor already closed"); + } + } +} + impl Drop for Dir { fn drop(&mut self) { + // dirfd isn't supported everywhere + #[cfg(not(any( + miri, + target_os = "redox", + target_os = "nto", + target_os = "vita", + target_os = "hurd", + target_os = "espidf", + target_os = "fuchsia", + target_os = "horizon", + )))] + { + let fd = unsafe { libc::dirfd(self.0) }; + debug_assert_fd_is_open(fd); + } let r = unsafe { libc::closedir(self.0) }; assert!( r == 0 || crate::io::Error::last_os_error().is_interrupted(), @@ -966,11 +962,6 @@ impl DirEntry { } #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "linux", target_os = "emscripten", target_os = "android", @@ -987,6 +978,7 @@ impl DirEntry { target_os = "aix", target_os = "nto", target_os = "hurd", + target_vendor = "apple", ))] pub fn ino(&self) -> u64 { self.entry.d_ino as u64 @@ -1003,15 +995,11 @@ impl DirEntry { } #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "netbsd", target_os = "openbsd", target_os = "freebsd", - target_os = "dragonfly" + target_os = "dragonfly", + target_vendor = "apple", ))] fn name_bytes(&self) -> &[u8] { use crate::slice; @@ -1023,15 +1011,11 @@ impl DirEntry { } } #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "netbsd", target_os = "openbsd", target_os = "freebsd", - target_os = "dragonfly" + target_os = "dragonfly", + target_vendor = "apple", )))] fn name_bytes(&self) -> &[u8] { self.name_cstr().to_bytes() @@ -1209,23 +1193,11 @@ impl File { cvt_r(|| unsafe { os_fsync(self.as_raw_fd()) })?; return Ok(()); - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - ))] + #[cfg(target_vendor = "apple")] unsafe fn os_fsync(fd: c_int) -> c_int { libc::fcntl(fd, libc::F_FULLFSYNC) } - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - )))] + #[cfg(not(target_vendor = "apple"))] unsafe fn os_fsync(fd: c_int) -> c_int { libc::fsync(fd) } @@ -1235,13 +1207,7 @@ impl File { cvt_r(|| unsafe { os_datasync(self.as_raw_fd()) })?; return Ok(()); - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - ))] + #[cfg(target_vendor = "apple")] unsafe fn os_datasync(fd: c_int) -> c_int { libc::fcntl(fd, libc::F_FULLFSYNC) } @@ -1260,16 +1226,12 @@ impl File { #[cfg(not(any( target_os = "android", target_os = "freebsd", - target_os = "ios", - target_os = "tvos", target_os = "linux", - target_os = "macos", target_os = "netbsd", target_os = "openbsd", - target_os = "watchos", - target_os = "visionos", target_os = "nto", target_os = "hurd", + target_vendor = "apple", )))] unsafe fn os_datasync(fd: c_int) -> c_int { libc::fsync(fd) @@ -1378,7 +1340,7 @@ impl File { io::ErrorKind::Unsupported, "setting file times not supported", )) - } else if #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos", target_os = "visionos"))] { + } else if #[cfg(target_vendor = "apple")] { let mut buf = [mem::MaybeUninit::<libc::timespec>::uninit(); 3]; let mut num_times = 0; let mut attrlist: libc::attrlist = unsafe { mem::zeroed() }; @@ -1531,11 +1493,11 @@ impl fmt::Debug for File { readlink(&p).ok() } - #[cfg(target_os = "macos")] + #[cfg(target_vendor = "apple")] fn get_path(fd: c_int) -> Option<PathBuf> { // FIXME: The use of PATH_MAX is generally not encouraged, but it - // is inevitable in this case because macOS defines `fcntl` with - // `F_GETPATH` in terms of `MAXPATHLEN`, and there are no + // is inevitable in this case because Apple targets define `fcntl` + // with `F_GETPATH` in terms of `MAXPATHLEN`, and there are no // alternatives. If a better method is invented, it should be used // instead. let mut buf = vec![0; libc::PATH_MAX as usize]; @@ -1576,12 +1538,12 @@ impl fmt::Debug for File { #[cfg(not(any( target_os = "linux", - target_os = "macos", target_os = "vxworks", all(target_os = "freebsd", target_arch = "x86_64"), target_os = "netbsd", target_os = "illumos", - target_os = "solaris" + target_os = "solaris", + target_vendor = "apple", )))] fn get_path(_fd: c_int) -> Option<PathBuf> { // FIXME(#24570): implement this for other Unix platforms @@ -1590,12 +1552,12 @@ impl fmt::Debug for File { #[cfg(any( target_os = "linux", - target_os = "macos", target_os = "freebsd", target_os = "hurd", target_os = "netbsd", target_os = "openbsd", - target_os = "vxworks" + target_os = "vxworks", + target_vendor = "apple", ))] fn get_mode(fd: c_int) -> Option<(bool, bool)> { let mode = unsafe { libc::fcntl(fd, libc::F_GETFL) }; @@ -1612,12 +1574,12 @@ impl fmt::Debug for File { #[cfg(not(any( target_os = "linux", - target_os = "macos", target_os = "freebsd", target_os = "hurd", target_os = "netbsd", target_os = "openbsd", - target_os = "vxworks" + target_os = "vxworks", + target_vendor = "apple", )))] fn get_mode(_fd: c_int) -> Option<(bool, bool)> { // FIXME(#24570): implement this for other Unix platforms @@ -1910,15 +1872,7 @@ fn open_to_and_set_permissions( Ok((writer, writer_metadata)) } -#[cfg(not(any( - target_os = "linux", - target_os = "android", - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", -)))] +#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))] pub fn copy(from: &Path, to: &Path) -> io::Result<u64> { let (mut reader, reader_metadata) = open_from(from)?; let (mut writer, _) = open_to_and_set_permissions(to, reader_metadata)?; @@ -1944,13 +1898,7 @@ pub fn copy(from: &Path, to: &Path) -> io::Result<u64> { } } -#[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" -))] +#[cfg(target_vendor = "apple")] pub fn copy(from: &Path, to: &Path) -> io::Result<u64> { use crate::sync::atomic::{AtomicBool, Ordering}; diff --git a/library/std/src/sys/pal/unix/kernel_copy.rs b/library/std/src/sys/pal/unix/kernel_copy.rs index 18acd5ecccd..cd38b7c07e2 100644 --- a/library/std/src/sys/pal/unix/kernel_copy.rs +++ b/library/std/src/sys/pal/unix/kernel_copy.rs @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> // We store the availability in a global to avoid unnecessary syscalls static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED); + let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { + NOT_PROBED => false, + UNAVAILABLE => return CopyResult::Fallback(0), + _ => true, + }; + syscall! { fn copy_file_range( fd_in: libc::c_int, @@ -571,25 +577,22 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> ) -> libc::ssize_t } - match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { - NOT_PROBED => { - // EPERM can indicate seccomp filters or an immutable file. - // To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported - // and some other error (ENOSYS or EPERM) if it's not available - let result = unsafe { - cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) - }; - - if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) { - HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); - } else { - HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed); - return CopyResult::Fallback(0); - } + fn probe_copy_file_range_support() -> u8 { + // In some cases, we cannot determine availability from the first + // `copy_file_range` call. In this case, we probe with an invalid file + // descriptor so that the results are easily interpretable. + match unsafe { + cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) + .map_err(|e| e.raw_os_error()) + } { + Err(Some(EPERM | ENOSYS)) => UNAVAILABLE, + Err(Some(EBADF)) => AVAILABLE, + Ok(_) => panic!("unexpected copy_file_range probe success"), + // Treat other errors as the syscall + // being unavailable. + Err(_) => UNAVAILABLE, } - UNAVAILABLE => return CopyResult::Fallback(0), - _ => {} - }; + } let mut written = 0u64; while written < max_len { @@ -604,6 +607,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0)) }; + if !have_probed && copy_result.is_ok() { + have_probed = true; + HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); + } + match copy_result { Ok(0) if written == 0 => { // fallback to work around several kernel bugs where copy_file_range will fail to @@ -619,7 +627,28 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> return match err.raw_os_error() { // when file offset + max_length > u64::MAX Some(EOVERFLOW) => CopyResult::Fallback(written), - Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => { + Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF)) + if written == 0 => + { + if !have_probed { + let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) { + // EPERM can indicate seccomp filters or an + // immutable file. To distinguish these + // cases we probe with invalid file + // descriptors which should result in EBADF + // if the syscall is supported and EPERM or + // ENOSYS if it's not available. + // + // For EOPNOTSUPP, see below. In the case of + // ENOSYS, we try to cover for faulty FUSE + // drivers. + probe_copy_file_range_support() + } else { + AVAILABLE + }; + HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed); + } + // Try fallback io::copy if either: // - Kernel version is < 4.5 (ENOSYS¹) // - Files are mounted on different fs (EXDEV) diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index 3ef43a923a3..735ed96bc7b 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -55,19 +55,19 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { // want! // // Hence, we set SIGPIPE to ignore when the program starts up in order - // to prevent this problem. Add `#[unix_sigpipe = "..."]` above `fn main()` to - // alter this behavior. + // to prevent this problem. Use `-Zon-broken-pipe=...` to alter this + // behavior. reset_sigpipe(sigpipe); stack_overflow::init(); args::init(argc, argv); // Normally, `thread::spawn` will call `Thread::set_name` but since this thread - // already exists, we have to call it ourselves. We only do this on macos + // already exists, we have to call it ourselves. We only do this on Apple targets // because some unix-like operating systems such as Linux share process-id and // thread-id for the main thread and so renaming the main thread will rename the // process and we only want to enable this on platforms we've tested. - if cfg!(target_os = "macos") { + if cfg!(target_vendor = "apple") { thread::Thread::set_name(&c"main"); } @@ -78,16 +78,12 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { target_os = "emscripten", target_os = "fuchsia", target_os = "vxworks", - // The poll on Darwin doesn't set POLLNVAL for closed fds. - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "redox", target_os = "l4re", target_os = "horizon", target_os = "vita", + // The poll on Darwin doesn't set POLLNVAL for closed fds. + target_vendor = "apple", )))] 'poll: { use crate::sys::os::errno; @@ -194,7 +190,7 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { _ => unreachable!(), }; if sigpipe_attr_specified { - UNIX_SIGPIPE_ATTR_SPECIFIED.store(true, crate::sync::atomic::Ordering::Relaxed); + ON_BROKEN_PIPE_FLAG_USED.store(true, crate::sync::atomic::Ordering::Relaxed); } if let Some(handler) = handler { rtassert!(signal(libc::SIGPIPE, handler) != libc::SIG_ERR); @@ -214,7 +210,7 @@ pub unsafe fn init(argc: isize, argv: *const *const u8, sigpipe: u8) { target_os = "fuchsia", target_os = "horizon", )))] -static UNIX_SIGPIPE_ATTR_SPECIFIED: crate::sync::atomic::AtomicBool = +static ON_BROKEN_PIPE_FLAG_USED: crate::sync::atomic::AtomicBool = crate::sync::atomic::AtomicBool::new(false); #[cfg(not(any( @@ -223,8 +219,8 @@ static UNIX_SIGPIPE_ATTR_SPECIFIED: crate::sync::atomic::AtomicBool = target_os = "fuchsia", target_os = "horizon", )))] -pub(crate) fn unix_sigpipe_attr_specified() -> bool { - UNIX_SIGPIPE_ATTR_SPECIFIED.load(crate::sync::atomic::Ordering::Relaxed) +pub(crate) fn on_broken_pipe_flag_used() -> bool { + ON_BROKEN_PIPE_FLAG_USED.load(crate::sync::atomic::Ordering::Relaxed) } // SAFETY: must be called only once during runtime cleanup. @@ -403,13 +399,12 @@ cfg_if::cfg_if! { // Use libumem for the (malloc-compatible) allocator #[link(name = "umem")] extern "C" {} - } else if #[cfg(target_os = "macos")] { - #[link(name = "System")] - extern "C" {} - } else if #[cfg(any(target_os = "ios", target_os = "tvos", target_os = "watchos", target_os = "visionos"))] { + } else if #[cfg(target_vendor = "apple")] { + // Link to `libSystem.dylib`. + // + // Don't get confused by the presence of `System.framework`, + // it is a deprecated wrapper over the dynamic library. #[link(name = "System")] - #[link(name = "objc")] - #[link(name = "Foundation", kind = "framework")] extern "C" {} } else if #[cfg(target_os = "fuchsia")] { #[link(name = "zircon")] diff --git a/library/std/src/sys/pal/unix/net.rs b/library/std/src/sys/pal/unix/net.rs index 7237989c905..b8dc1538a63 100644 --- a/library/std/src/sys/pal/unix/net.rs +++ b/library/std/src/sys/pal/unix/net.rs @@ -86,7 +86,14 @@ impl Socket { // flag to atomically create the socket and set it as // CLOEXEC. On Linux this was added in 2.6.27. let fd = cvt(libc::socket(fam, ty | libc::SOCK_CLOEXEC, 0))?; - Ok(Socket(FileDesc::from_raw_fd(fd))) + let socket = Socket(FileDesc::from_raw_fd(fd)); + + // DragonFlyBSD, FreeBSD and NetBSD use `SO_NOSIGPIPE` as a `setsockopt` + // flag to disable `SIGPIPE` emission on socket. + #[cfg(any(target_os = "freebsd", target_os = "netbsd", target_os = "dragonfly"))] + setsockopt(&socket, libc::SOL_SOCKET, libc::SO_NOSIGPIPE, 1)?; + + Ok(socket) } else { let fd = cvt(libc::socket(fam, ty, 0))?; let fd = FileDesc::from_raw_fd(fd); diff --git a/library/std/src/sys/pal/unix/os.rs b/library/std/src/sys/pal/unix/os.rs index 96492fedece..8afc49f5227 100644 --- a/library/std/src/sys/pal/unix/os.rs +++ b/library/std/src/sys/pal/unix/os.rs @@ -63,17 +63,7 @@ extern "C" { )] #[cfg_attr(any(target_os = "solaris", target_os = "illumos"), link_name = "___errno")] #[cfg_attr(target_os = "nto", link_name = "__get_errno_ptr")] - #[cfg_attr( - any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "freebsd", - target_os = "watchos", - target_os = "visionos", - ), - link_name = "__error" - )] + #[cfg_attr(any(target_os = "freebsd", target_vendor = "apple"), link_name = "__error")] #[cfg_attr(target_os = "haiku", link_name = "_errnop")] #[cfg_attr(target_os = "aix", link_name = "_Errno")] fn errno_location() -> *mut c_int; @@ -431,13 +421,7 @@ pub fn current_exe() -> io::Result<PathBuf> { Ok(PathBuf::from(OsString::from_vec(e))) } -#[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" -))] +#[cfg(target_vendor = "apple")] pub fn current_exe() -> io::Result<PathBuf> { unsafe { let mut sz: u32 = 0; @@ -592,12 +576,36 @@ impl Iterator for Env { } } -#[cfg(target_os = "macos")] +// Use `_NSGetEnviron` on Apple platforms. +// +// `_NSGetEnviron` is the documented alternative (see `man environ`), and has +// been available since the first versions of both macOS and iOS. +// +// Nowadays, specifically since macOS 10.8, `environ` has been exposed through +// `libdyld.dylib`, which is linked via. `libSystem.dylib`: +// <https://github.com/apple-oss-distributions/dyld/blob/dyld-1160.6/libdyld/libdyldGlue.cpp#L913> +// +// So in the end, it likely doesn't really matter which option we use, but the +// performance cost of using `_NSGetEnviron` is extremely miniscule, and it +// might be ever so slightly more supported, so let's just use that. +// +// NOTE: The header where this is defined (`crt_externs.h`) was added to the +// iOS 13.0 SDK, which has been the source of a great deal of confusion in the +// past about the availability of this API. +// +// NOTE(madsmtm): Neither this nor using `environ` has been verified to not +// cause App Store rejections; if this is found to be the case, an alternative +// implementation of this is possible using `[NSProcessInfo environment]` +// - which internally uses `_NSGetEnviron` and a system-wide lock on the +// environment variables to protect against `setenv`, so using that might be +// desirable anyhow? Though it also means that we have to link to Foundation. +#[cfg(target_vendor = "apple")] pub unsafe fn environ() -> *mut *const *const c_char { libc::_NSGetEnviron() as *mut *const *const c_char } -#[cfg(not(target_os = "macos"))] +// Use the `environ` static which is part of POSIX. +#[cfg(not(target_vendor = "apple"))] pub unsafe fn environ() -> *mut *const *const c_char { extern "C" { static mut environ: *const *const c_char; @@ -703,32 +711,26 @@ pub fn home_dir() -> Option<PathBuf> { #[cfg(any( target_os = "android", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "emscripten", target_os = "redox", target_os = "vxworks", target_os = "espidf", target_os = "horizon", target_os = "vita", + all(target_vendor = "apple", not(target_os = "macos")), ))] unsafe fn fallback() -> Option<OsString> { None } #[cfg(not(any( target_os = "android", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "emscripten", target_os = "redox", target_os = "vxworks", target_os = "espidf", target_os = "horizon", target_os = "vita", + all(target_vendor = "apple", not(target_os = "macos")), )))] unsafe fn fallback() -> Option<OsString> { let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) { diff --git a/library/std/src/sys/pal/unix/process/process_unix.rs b/library/std/src/sys/pal/unix/process/process_unix.rs index d65657790c4..e2fca8c7e63 100644 --- a/library/std/src/sys/pal/unix/process/process_unix.rs +++ b/library/std/src/sys/pal/unix/process/process_unix.rs @@ -353,11 +353,11 @@ impl Command { // Inherit the signal mask from the parent rather than resetting it (i.e. do not call // pthread_sigmask). - // If #[unix_sigpipe] is specified, don't reset SIGPIPE to SIG_DFL. - // If #[unix_sigpipe] is not specified, reset SIGPIPE to SIG_DFL for backward compatibility. + // If -Zon-broken-pipe is used, don't reset SIGPIPE to SIG_DFL. + // If -Zon-broken-pipe is not used, reset SIGPIPE to SIG_DFL for backward compatibility. // - // #[unix_sigpipe] is an opportunity to change the default here. - if !crate::sys::pal::unix_sigpipe_attr_specified() { + // -Zon-broken-pipe is an opportunity to change the default here. + if !crate::sys::pal::on_broken_pipe_flag_used() { #[cfg(target_os = "android")] // see issue #88585 { let mut action: libc::sigaction = mem::zeroed(); @@ -420,13 +420,11 @@ impl Command { } #[cfg(not(any( - target_os = "macos", - target_os = "tvos", - target_os = "watchos", target_os = "freebsd", all(target_os = "linux", target_env = "gnu"), all(target_os = "linux", target_env = "musl"), target_os = "nto", + target_vendor = "apple", )))] fn posix_spawn( &mut self, @@ -439,14 +437,11 @@ impl Command { // Only support platforms for which posix_spawn() can return ENOENT // directly. #[cfg(any( - target_os = "macos", - // FIXME: `target_os = "ios"`? - target_os = "tvos", - target_os = "watchos", target_os = "freebsd", all(target_os = "linux", target_env = "gnu"), all(target_os = "linux", target_env = "musl"), target_os = "nto", + target_vendor = "apple", ))] fn posix_spawn( &mut self, @@ -455,7 +450,7 @@ impl Command { ) -> io::Result<Option<Process>> { use crate::mem::MaybeUninit; use crate::sys::weak::weak; - use crate::sys::{self, cvt_nz, unix_sigpipe_attr_specified}; + use crate::sys::{self, cvt_nz, on_broken_pipe_flag_used}; if self.get_gid().is_some() || self.get_uid().is_some() @@ -530,7 +525,7 @@ impl Command { } let addchdir = match self.get_cwd() { Some(cwd) => { - if cfg!(any(target_os = "macos", target_os = "tvos", target_os = "watchos")) { + if cfg!(target_vendor = "apple") { // There is a bug in macOS where a relative executable // path like "../myprogram" will cause `posix_spawn` to // successfully launch the program, but erroneously return @@ -617,11 +612,11 @@ impl Command { // Inherit the signal mask from this process rather than resetting it (i.e. do not call // posix_spawnattr_setsigmask). - // If #[unix_sigpipe] is specified, don't reset SIGPIPE to SIG_DFL. - // If #[unix_sigpipe] is not specified, reset SIGPIPE to SIG_DFL for backward compatibility. + // If -Zon-broken-pipe is used, don't reset SIGPIPE to SIG_DFL. + // If -Zon-broken-pipe is not used, reset SIGPIPE to SIG_DFL for backward compatibility. // - // #[unix_sigpipe] is an opportunity to change the default here. - if !unix_sigpipe_attr_specified() { + // -Zon-broken-pipe is an opportunity to change the default here. + if !on_broken_pipe_flag_used() { let mut default_set = MaybeUninit::<libc::sigset_t>::uninit(); cvt(sigemptyset(default_set.as_mut_ptr()))?; cvt(sigaddset(default_set.as_mut_ptr(), libc::SIGPIPE))?; @@ -1040,24 +1035,20 @@ fn signal_string(signal: i32) -> &'static str { #[cfg(any(target_os = "linux", target_os = "nto"))] libc::SIGPWR => " (SIGPWR)", #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly", target_os = "nto", + target_vendor = "apple", ))] libc::SIGEMT => " (SIGEMT)", #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", - target_os = "dragonfly" + target_os = "dragonfly", + target_vendor = "apple", ))] libc::SIGINFO => " (SIGINFO)", #[cfg(target_os = "hurd")] diff --git a/library/std/src/sys/pal/unix/rand.rs b/library/std/src/sys/pal/unix/rand.rs index de087d98eb8..e6df109a6b8 100644 --- a/library/std/src/sys/pal/unix/rand.rs +++ b/library/std/src/sys/pal/unix/rand.rs @@ -12,11 +12,6 @@ pub fn hashmap_random_keys() -> (u64, u64) { #[cfg(all( unix, - not(target_os = "macos"), - not(target_os = "ios"), - not(target_os = "tvos"), - not(target_os = "watchos"), - not(target_os = "visionos"), not(target_os = "openbsd"), not(target_os = "netbsd"), not(target_os = "fuchsia"), @@ -24,6 +19,7 @@ pub fn hashmap_random_keys() -> (u64, u64) { not(target_os = "vxworks"), not(target_os = "emscripten"), not(target_os = "vita"), + not(target_vendor = "apple"), ))] mod imp { use crate::fs::File; @@ -63,7 +59,14 @@ mod imp { unsafe { getrandom(buf.as_mut_ptr().cast(), buf.len(), libc::GRND_NONBLOCK) } } - #[cfg(any(target_os = "espidf", target_os = "horizon", target_os = "freebsd", netbsd10))] + #[cfg(any( + target_os = "espidf", + target_os = "horizon", + target_os = "freebsd", + netbsd10, + target_os = "illumos", + target_os = "solaris" + ))] fn getrandom(buf: &mut [u8]) -> libc::ssize_t { unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) } } @@ -87,6 +90,8 @@ mod imp { target_os = "horizon", target_os = "freebsd", target_os = "dragonfly", + target_os = "solaris", + target_os = "illumos", netbsd10 )))] fn getrandom_fill_bytes(_buf: &mut [u8]) -> bool { @@ -100,6 +105,8 @@ mod imp { target_os = "horizon", target_os = "freebsd", target_os = "dragonfly", + target_os = "solaris", + target_os = "illumos", netbsd10 ))] fn getrandom_fill_bytes(v: &mut [u8]) -> bool { diff --git a/library/std/src/sys/pal/unix/stack_overflow.rs b/library/std/src/sys/pal/unix/stack_overflow.rs index 26c49257ad0..2e5bd85327a 100644 --- a/library/std/src/sys/pal/unix/stack_overflow.rs +++ b/library/std/src/sys/pal/unix/stack_overflow.rs @@ -491,6 +491,14 @@ mod imp { } } +// This is intentionally not enabled on iOS/tvOS/watchOS/visionOS, as it uses +// several symbols that might lead to rejections from the App Store, namely +// `sigaction`, `sigaltstack`, `sysctlbyname`, `mmap`, `munmap` and `mprotect`. +// +// This might be overly cautious, though it is also what Swift does (and they +// usually have fewer qualms about forwards compatibility, since the runtime +// is shipped with the OS): +// <https://github.com/apple/swift/blob/swift-5.10-RELEASE/stdlib/public/runtime/CrashHandlerMacOS.cpp> #[cfg(not(any( target_os = "linux", target_os = "freebsd", diff --git a/library/std/src/sys/pal/unix/thread.rs b/library/std/src/sys/pal/unix/thread.rs index 6a6bfc77a85..853ef8736de 100644 --- a/library/std/src/sys/pal/unix/thread.rs +++ b/library/std/src/sys/pal/unix/thread.rs @@ -150,13 +150,7 @@ impl Thread { } } - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" - ))] + #[cfg(target_vendor = "apple")] pub fn set_name(name: &CStr) { unsafe { let name = truncate_cstr::<{ libc::MAXTHREADNAMESIZE }>(name); @@ -301,14 +295,10 @@ impl Drop for Thread { #[cfg(any( target_os = "linux", - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "nto", target_os = "solaris", target_os = "illumos", + target_vendor = "apple", ))] fn truncate_cstr<const MAX_WITH_NUL: usize>(cstr: &CStr) -> [libc::c_char; MAX_WITH_NUL] { let mut result = [0; MAX_WITH_NUL]; @@ -325,11 +315,9 @@ pub fn available_parallelism() -> io::Result<NonZero<usize>> { target_os = "emscripten", target_os = "fuchsia", target_os = "hurd", - target_os = "ios", - target_os = "tvos", target_os = "linux", - target_os = "macos", target_os = "aix", + target_vendor = "apple", ))] { #[allow(unused_assignments)] #[allow(unused_mut)] diff --git a/library/std/src/sys/pal/unix/thread_local_dtor.rs b/library/std/src/sys/pal/unix/thread_local_dtor.rs index 1164c1782f4..75db6e112ed 100644 --- a/library/std/src/sys/pal/unix/thread_local_dtor.rs +++ b/library/std/src/sys/pal/unix/thread_local_dtor.rs @@ -76,13 +76,7 @@ pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { // workaround below is to register, via _tlv_atexit, a custom DTOR list once per // thread. thread_local dtors are pushed to the DTOR list without calling // _tlv_atexit. -#[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" -))] +#[cfg(target_vendor = "apple")] pub unsafe fn register_dtor(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { use crate::cell::{Cell, RefCell}; use crate::ptr; diff --git a/library/std/src/sys/pal/unix/thread_parking/netbsd.rs b/library/std/src/sys/pal/unix/thread_parking.rs index 5eeb37f8763..1366410b71e 100644 --- a/library/std/src/sys/pal/unix/thread_parking/netbsd.rs +++ b/library/std/src/sys/pal/unix/thread_parking.rs @@ -1,7 +1,11 @@ +// Only used on NetBSD. If other platforms start using id-based parking, use +// separate modules for each platform. +#![cfg(target_os = "netbsd")] + use crate::ffi::{c_int, c_void}; use crate::ptr; use crate::time::Duration; -use libc::{_lwp_self, clockid_t, lwpid_t, time_t, timespec, CLOCK_MONOTONIC}; +use libc::{_lwp_self, c_long, clockid_t, lwpid_t, time_t, timespec, CLOCK_MONOTONIC}; extern "C" { fn ___lwp_park60( @@ -34,7 +38,7 @@ pub fn park_timeout(dur: Duration, hint: usize) { // Saturate so that the operation will definitely time out // (even if it is after the heat death of the universe). tv_sec: dur.as_secs().try_into().ok().unwrap_or(time_t::MAX), - tv_nsec: dur.subsec_nanos().into(), + tv_nsec: dur.subsec_nanos() as c_long, }; // Timeout needs to be mutable since it is modified on NetBSD 9.0 and diff --git a/library/std/src/sys/pal/unix/thread_parking/mod.rs b/library/std/src/sys/pal/unix/thread_parking/mod.rs deleted file mode 100644 index 3348d4b366d..00000000000 --- a/library/std/src/sys/pal/unix/thread_parking/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Thread parking on systems without futex support. - -#![cfg(not(any( - target_os = "linux", - target_os = "android", - all(target_os = "emscripten", target_feature = "atomics"), - target_os = "freebsd", - target_os = "openbsd", - target_os = "dragonfly", - target_os = "fuchsia", -)))] - -cfg_if::cfg_if! { - if #[cfg(all( - any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos", - ), - not(miri), - ))] { - mod darwin; - pub use darwin::Parker; - } else if #[cfg(target_os = "netbsd")] { - mod netbsd; - pub use netbsd::{current, park, park_timeout, unpark, ThreadId}; - } else { - mod pthread; - pub use pthread::Parker; - } -} diff --git a/library/std/src/sys/pal/unix/time.rs b/library/std/src/sys/pal/unix/time.rs index e69775be8cf..535fe6b27d9 100644 --- a/library/std/src/sys/pal/unix/time.rs +++ b/library/std/src/sys/pal/unix/time.rs @@ -86,13 +86,7 @@ impl Timespec { // Please note that Apple OS nonetheless accepts the standard unix format when // setting file times, which makes this compensation round-trippable and generally // transparent. - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - ))] + #[cfg(target_vendor = "apple")] let (tv_sec, tv_nsec) = if (tv_sec <= 0 && tv_sec > i64::MIN) && (tv_nsec < 0 && tv_nsec > -1_000_000_000) { (tv_sec - 1, tv_nsec + 1_000_000_000) @@ -275,21 +269,9 @@ impl Instant { // // Instant on macos was historically implemented using mach_absolute_time; // we preserve this value domain out of an abundance of caution. - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" - ))] + #[cfg(target_vendor = "apple")] const clock_id: libc::clockid_t = libc::CLOCK_UPTIME_RAW; - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "watchos", - target_os = "visionos", - target_os = "tvos" - )))] + #[cfg(not(target_vendor = "apple"))] const clock_id: libc::clockid_t = libc::CLOCK_MONOTONIC; Instant { t: Timespec::now(clock_id) } } diff --git a/library/std/src/sys/pal/unix/weak.rs b/library/std/src/sys/pal/unix/weak.rs index 48cc8633e93..4765a286e6b 100644 --- a/library/std/src/sys/pal/unix/weak.rs +++ b/library/std/src/sys/pal/unix/weak.rs @@ -28,7 +28,7 @@ use crate::ptr; use crate::sync::atomic::{self, AtomicPtr, Ordering}; // We can use true weak linkage on ELF targets. -#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] +#[cfg(all(unix, not(target_vendor = "apple")))] pub(crate) macro weak { (fn $name:ident($($t:ty),*) -> $ret:ty) => ( let ref $name: ExternWeak<unsafe extern "C" fn($($t),*) -> $ret> = { @@ -43,7 +43,7 @@ pub(crate) macro weak { } // On non-ELF targets, use the dlsym approximation of weak linkage. -#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))] +#[cfg(target_vendor = "apple")] pub(crate) use self::dlsym as weak; pub(crate) struct ExternWeak<F: Copy> { diff --git a/library/std/src/sys/pal/unsupported/mod.rs b/library/std/src/sys/pal/unsupported/mod.rs index be344fb7cae..01f5cfd4297 100644 --- a/library/std/src/sys/pal/unsupported/mod.rs +++ b/library/std/src/sys/pal/unsupported/mod.rs @@ -14,7 +14,6 @@ pub mod thread; #[cfg(target_thread_local)] pub mod thread_local_dtor; pub mod thread_local_key; -pub mod thread_parking; pub mod time; mod common; diff --git a/library/std/src/sys/pal/wasi/mod.rs b/library/std/src/sys/pal/wasi/mod.rs index a78547261ad..c1266619b36 100644 --- a/library/std/src/sys/pal/wasi/mod.rs +++ b/library/std/src/sys/pal/wasi/mod.rs @@ -39,13 +39,6 @@ pub mod thread_local_dtor; pub mod thread_local_key; pub mod time; -cfg_if::cfg_if! { - if #[cfg(not(target_feature = "atomics"))] { - #[path = "../unsupported/thread_parking.rs"] - pub mod thread_parking; - } -} - #[path = "../unsupported/common.rs"] #[deny(unsafe_op_in_unsafe_fn)] #[allow(unused)] diff --git a/library/std/src/sys/pal/wasip2/mod.rs b/library/std/src/sys/pal/wasip2/mod.rs index 94aa458d2f9..6787ffb4bed 100644 --- a/library/std/src/sys/pal/wasip2/mod.rs +++ b/library/std/src/sys/pal/wasip2/mod.rs @@ -41,15 +41,6 @@ pub mod thread_local_key; #[path = "../wasi/time.rs"] pub mod time; -cfg_if::cfg_if! { - if #[cfg(target_feature = "atomics")] { - compile_error!("The wasm32-wasip2 target does not support atomics"); - } else { - #[path = "../unsupported/thread_parking.rs"] - pub mod thread_parking; - } -} - #[path = "../unsupported/common.rs"] #[deny(unsafe_op_in_unsafe_fn)] #[allow(unused)] diff --git a/library/std/src/sys/pal/wasm/mod.rs b/library/std/src/sys/pal/wasm/mod.rs index 5cbc3e45341..75dd10826cc 100644 --- a/library/std/src/sys/pal/wasm/mod.rs +++ b/library/std/src/sys/pal/wasm/mod.rs @@ -50,8 +50,6 @@ cfg_if::cfg_if! { } else { #[path = "../unsupported/thread.rs"] pub mod thread; - #[path = "../unsupported/thread_parking.rs"] - pub mod thread_parking; } } diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index 1c828bac4b6..9d58ce05f01 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -357,7 +357,19 @@ compat_fn_with_fallback! { } #[cfg(not(target_vendor = "win7"))] -#[link(name = "synchronization")] +// Use raw-dylib to import synchronization functions to workaround issues with the older mingw import library. +#[cfg_attr( + target_arch = "x86", + link( + name = "api-ms-win-core-synch-l1-2-0", + kind = "raw-dylib", + import_name_type = "undecorated" + ) +)] +#[cfg_attr( + not(target_arch = "x86"), + link(name = "api-ms-win-core-synch-l1-2-0", kind = "raw-dylib") +)] extern "system" { pub fn WaitOnAddress( address: *const c_void, diff --git a/library/std/src/sys/pal/windows/c/windows_sys.rs b/library/std/src/sys/pal/windows/c/windows_sys.rs index d180122d735..1da8871ae44 100644 --- a/library/std/src/sys/pal/windows/c/windows_sys.rs +++ b/library/std/src/sys/pal/windows/c/windows_sys.rs @@ -1,4 +1,4 @@ -// Bindings generated by `windows-bindgen` 0.55.0 +// Bindings generated by `windows-bindgen` 0.56.0 #![allow(non_snake_case, non_upper_case_globals, non_camel_case_types, dead_code, clippy::all)] #[link(name = "advapi32")] @@ -345,7 +345,7 @@ extern "system" { } #[link(name = "kernel32")] extern "system" { - pub fn GetSystemTimePreciseAsFileTime(lpsystemtimeasfiletime: *mut FILETIME) -> (); + pub fn GetSystemTimePreciseAsFileTime(lpsystemtimeasfiletime: *mut FILETIME); } #[link(name = "kernel32")] extern "system" { @@ -1018,7 +1018,7 @@ impl Clone for CONTEXT_0_0 { } } #[repr(C)] -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] pub struct CONTEXT { pub P1Home: u64, pub P2Home: u64, @@ -1067,30 +1067,30 @@ pub struct CONTEXT { pub LastExceptionToRip: u64, pub LastExceptionFromRip: u64, } -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for CONTEXT {} -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for CONTEXT { fn clone(&self) -> Self { *self } } #[repr(C)] -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] pub union CONTEXT_0 { pub FltSave: XSAVE_FORMAT, pub Anonymous: CONTEXT_0_0, } -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for CONTEXT_0 {} -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for CONTEXT_0 { fn clone(&self) -> Self { *self } } #[repr(C)] -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] pub struct CONTEXT_0_0 { pub Header: [M128A; 2], pub Legacy: [M128A; 8], @@ -1111,9 +1111,9 @@ pub struct CONTEXT_0_0 { pub Xmm14: M128A, pub Xmm15: M128A, } -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for CONTEXT_0_0 {} -#[cfg(target_arch = "x86_64")] +#[cfg(any(target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for CONTEXT_0_0 { fn clone(&self) -> Self { *self @@ -3339,7 +3339,7 @@ pub const FILE_WRITE_EA: FILE_ACCESS_RIGHTS = 16u32; pub const FILE_WRITE_THROUGH: NTCREATEFILE_CREATE_OPTIONS = 2u32; pub const FIONBIO: i32 = -2147195266i32; #[repr(C)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] pub struct FLOATING_SAVE_AREA { pub ControlWord: u32, pub StatusWord: u32, @@ -3351,9 +3351,9 @@ pub struct FLOATING_SAVE_AREA { pub RegisterArea: [u8; 80], pub Cr0NpxState: u32, } -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for FLOATING_SAVE_AREA {} -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for FLOATING_SAVE_AREA { fn clone(&self) -> Self { *self @@ -4106,7 +4106,7 @@ impl Clone for WSABUF { } } #[repr(C)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] pub struct WSADATA { pub wVersion: u16, pub wHighVersion: u16, @@ -4116,9 +4116,9 @@ pub struct WSADATA { pub szDescription: [i8; 257], pub szSystemStatus: [i8; 129], } -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for WSADATA {} -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for WSADATA { fn clone(&self) -> Self { *self @@ -4286,7 +4286,7 @@ pub const WSA_SECURE_HOST_NOT_FOUND: WSA_ERROR = 11032i32; pub const WSA_WAIT_EVENT_0: WSA_ERROR = 0i32; pub const WSA_WAIT_IO_COMPLETION: WSA_ERROR = 192i32; #[repr(C)] -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] pub struct XSAVE_FORMAT { pub ControlWord: u16, pub StatusWord: u16, @@ -4305,9 +4305,9 @@ pub struct XSAVE_FORMAT { pub XmmRegisters: [M128A; 16], pub Reserved4: [u8; 96], } -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Copy for XSAVE_FORMAT {} -#[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] +#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "x86_64"))] impl Clone for XSAVE_FORMAT { fn clone(&self) -> Self { *self diff --git a/library/std/src/sys/pal/windows/mod.rs b/library/std/src/sys/pal/windows/mod.rs index a734c2bd4c7..402a205977b 100644 --- a/library/std/src/sys/pal/windows/mod.rs +++ b/library/std/src/sys/pal/windows/mod.rs @@ -33,7 +33,6 @@ pub mod stdio; pub mod thread; pub mod thread_local_dtor; pub mod thread_local_key; -pub mod thread_parking; pub mod time; cfg_if::cfg_if! { if #[cfg(not(target_vendor = "uwp"))] { @@ -201,14 +200,21 @@ pub fn to_u16s<S: AsRef<OsStr>>(s: S) -> crate::io::Result<Vec<u16>> { // currently reside in the buffer. This function is an abstraction over these // functions by making them easier to call. // -// The first callback, `f1`, is yielded a (pointer, len) pair which can be +// The first callback, `f1`, is passed a (pointer, len) pair which can be // passed to a syscall. The `ptr` is valid for `len` items (u16 in this case). -// The closure is expected to return what the syscall returns which will be -// interpreted by this function to determine if the syscall needs to be invoked -// again (with more buffer space). +// The closure is expected to: +// - On success, return the actual length of the written data *without* the null terminator. +// This can be 0. In this case the last_error must be left unchanged. +// - On insufficient buffer space, +// - either return the required length *with* the null terminator, +// - or set the last-error to ERROR_INSUFFICIENT_BUFFER and return `len`. +// - On other failure, return 0 and set last_error. +// +// This is how most but not all syscalls indicate the required buffer space. +// Other syscalls may need translation to match this protocol. // // Once the syscall has completed (errors bail out early) the second closure is -// yielded the data which has been read from the syscall. The return value +// passed the data which has been read from the syscall. The return value // from this closure is then the return value of the function. pub fn fill_utf16_buf<F1, F2, T>(mut f1: F1, f2: F2) -> crate::io::Result<T> where @@ -332,7 +338,7 @@ pub fn abort_internal() -> ! { core::arch::asm!("int $$0x29", in("ecx") c::FAST_FAIL_FATAL_APP_EXIT, options(noreturn, nostack)); } else if #[cfg(all(target_arch = "arm", target_feature = "thumb-mode"))] { core::arch::asm!(".inst 0xDEFB", in("r0") c::FAST_FAIL_FATAL_APP_EXIT, options(noreturn, nostack)); - } else if #[cfg(target_arch = "aarch64")] { + } else if #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] { core::arch::asm!("brk 0xF003", in("x0") c::FAST_FAIL_FATAL_APP_EXIT, options(noreturn, nostack)); } else { core::intrinsics::abort(); diff --git a/library/std/src/sys/pal/windows/os.rs b/library/std/src/sys/pal/windows/os.rs index 374c9845ea4..64d8b72aed2 100644 --- a/library/std/src/sys/pal/windows/os.rs +++ b/library/std/src/sys/pal/windows/os.rs @@ -326,6 +326,8 @@ fn home_dir_crt() -> Option<PathBuf> { super::fill_utf16_buf( |buf, mut sz| { + // GetUserProfileDirectoryW does not quite use the usual protocol for + // negotiating the buffer size, so we have to translate. match c::GetUserProfileDirectoryW( ptr::without_provenance_mut(CURRENT_PROCESS_TOKEN), buf, diff --git a/library/std/src/sys/pal/windows/thread_local_key.rs b/library/std/src/sys/pal/windows/thread_local_key.rs index 4c00860dae3..e5ba619fc6b 100644 --- a/library/std/src/sys/pal/windows/thread_local_key.rs +++ b/library/std/src/sys/pal/windows/thread_local_key.rs @@ -141,9 +141,15 @@ impl StaticKey { panic!("out of TLS indexes"); } - self.key.store(key + 1, Release); register_dtor(self); + // Release-storing the key needs to be the last thing we do. + // This is because in `fn key()`, other threads will do an acquire load of the key, + // and if that sees this write then it will entirely bypass the `InitOnce`. We thus + // need to establish synchronization through `key`. In particular that acquire load + // must happen-after the register_dtor above, to ensure the dtor actually runs! + self.key.store(key + 1, Release); + let r = c::InitOnceComplete(self.once.get(), 0, ptr::null_mut()); debug_assert_eq!(r, c::TRUE); @@ -313,8 +319,22 @@ unsafe fn run_dtors() { // Use acquire ordering to observe key initialization. let mut cur = DTORS.load(Acquire); while !cur.is_null() { - let key = (*cur).key.load(Relaxed) - 1; + let pre_key = (*cur).key.load(Acquire); let dtor = (*cur).dtor.unwrap(); + cur = (*cur).next.load(Relaxed); + + // In StaticKey::init, we register the dtor before setting `key`. + // So if one thread's `run_dtors` races with another thread executing `init` on the same + // `StaticKey`, we can encounter a key of 0 here. That means this key was never + // initialized in this thread so we can safely skip it. + if pre_key == 0 { + continue; + } + // If this is non-zero, then via the `Acquire` load above we synchronized with + // everything relevant for this key. (It's not clear that this is needed, since the + // release-acquire pair on DTORS also establishes synchronization, but better safe than + // sorry.) + let key = pre_key - 1; let ptr = c::TlsGetValue(key); if !ptr.is_null() { @@ -322,8 +342,6 @@ unsafe fn run_dtors() { dtor(ptr as *mut _); any_run = true; } - - cur = (*cur).next.load(Relaxed); } if !any_run { diff --git a/library/std/src/sys/pal/xous/mod.rs b/library/std/src/sys/pal/xous/mod.rs index 7914a255aea..68189bcc2e3 100644 --- a/library/std/src/sys/pal/xous/mod.rs +++ b/library/std/src/sys/pal/xous/mod.rs @@ -18,7 +18,6 @@ pub mod process; pub mod stdio; pub mod thread; pub mod thread_local_key; -pub mod thread_parking; pub mod time; #[path = "../unsupported/common.rs"] diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs index 4f79f8c4961..0b22eabca6d 100644 --- a/library/std/src/sys/pal/zkvm/mod.rs +++ b/library/std/src/sys/pal/zkvm/mod.rs @@ -32,9 +32,6 @@ pub mod time; #[path = "../unsupported/thread.rs"] pub mod thread; -#[path = "../unsupported/thread_parking.rs"] -pub mod thread_parking; - mod abi; use crate::io as std_io; diff --git a/library/std/src/sys/personality/dwarf/eh.rs b/library/std/src/sys/personality/dwarf/eh.rs index 3f3615ea3e0..ff88ef4e0e1 100644 --- a/library/std/src/sys/personality/dwarf/eh.rs +++ b/library/std/src/sys/personality/dwarf/eh.rs @@ -54,7 +54,14 @@ pub enum EHAction { Terminate, } -pub const USING_SJLJ_EXCEPTIONS: bool = cfg!(all(target_os = "ios", target_arch = "arm")); +/// 32-bit Apple ARM uses SjLj exceptions, except for watchOS. +/// +/// I.e. iOS and tvOS, as those are the only Apple OSes that used 32-bit ARM +/// devices. +/// +/// <https://github.com/llvm/llvm-project/blob/llvmorg-18.1.4/clang/lib/Driver/ToolChains/Darwin.cpp#L3107-L3119> +pub const USING_SJLJ_EXCEPTIONS: bool = + cfg!(all(target_vendor = "apple", not(target_os = "watchos"), target_arch = "arm")); pub unsafe fn find_eh_action(lsda: *const u8, context: &EHContext<'_>) -> Result<EHAction, ()> { if lsda.is_null() { diff --git a/library/std/src/sys/personality/gcc.rs b/library/std/src/sys/personality/gcc.rs index b0f744dd966..0dc53550ca9 100644 --- a/library/std/src/sys/personality/gcc.rs +++ b/library/std/src/sys/personality/gcc.rs @@ -92,11 +92,12 @@ const UNWIND_DATA_REG: (i32, i32) = (4, 5); // a0, a1 // https://github.com/gcc-mirror/gcc/blob/trunk/libgcc/unwind-c.c cfg_if::cfg_if! { - if #[cfg(all(target_arch = "arm", not(target_os = "ios"), not(target_os = "tvos"), not(target_os = "watchos"), not(target_os = "visionos"), not(target_os = "netbsd")))] { + if #[cfg(all(not(all(target_vendor = "apple", not(target_os = "watchos"))), target_arch = "arm", not(target_os = "netbsd")))] { // ARM EHABI personality routine. // https://web.archive.org/web/20190728160938/https://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf // - // iOS uses the default routine instead since it uses SjLj unwinding. + // Apple 32-bit ARM (but not watchOS) uses the default routine instead + // since it uses SjLj unwinding. #[lang = "eh_personality"] unsafe extern "C" fn rust_eh_personality( state: uw::_Unwind_State, diff --git a/library/std/src/sys/sync/condvar/pthread.rs b/library/std/src/sys/sync/condvar/pthread.rs index 0475f985078..a2a96410d93 100644 --- a/library/std/src/sys/sync/condvar/pthread.rs +++ b/library/std/src/sys/sync/condvar/pthread.rs @@ -30,14 +30,10 @@ impl LazyInit for AllocatedCondvar { cfg_if::cfg_if! { if #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "l4re", target_os = "android", - target_os = "redox" + target_os = "redox", + target_vendor = "apple", ))] { // `pthread_condattr_setclock` is unfortunately not supported on these platforms. } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { @@ -124,14 +120,10 @@ impl Condvar { // default system clock). This approach avoids all problems that result // from changes made to the system time. #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "android", target_os = "espidf", - target_os = "horizon" + target_os = "horizon", + target_vendor = "apple", )))] pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { use crate::sys::time::Timespec; @@ -160,14 +152,10 @@ impl Condvar { // https://github.com/llvm-mirror/libcxx/blob/release_35/src/condition_variable.cpp#L46 // https://github.com/llvm-mirror/libcxx/blob/release_35/include/__mutex_base#L367 #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "android", target_os = "espidf", - target_os = "horizon" + target_os = "horizon", + target_vendor = "apple", ))] pub unsafe fn wait_timeout(&self, mutex: &Mutex, dur: Duration) -> bool { use crate::sys::time::SystemTime; diff --git a/library/std/src/sys/sync/mod.rs b/library/std/src/sys/sync/mod.rs index 623e6bccd51..52fac5902a2 100644 --- a/library/std/src/sys/sync/mod.rs +++ b/library/std/src/sys/sync/mod.rs @@ -2,8 +2,10 @@ mod condvar; mod mutex; mod once; mod rwlock; +mod thread_parking; pub use condvar::Condvar; pub use mutex::Mutex; pub use once::{Once, OnceState}; pub use rwlock::RwLock; +pub use thread_parking::Parker; diff --git a/library/std/src/sys/pal/unix/thread_parking/darwin.rs b/library/std/src/sys/sync/thread_parking/darwin.rs index 8231f3cba2d..973c08f0317 100644 --- a/library/std/src/sys/pal/unix/thread_parking/darwin.rs +++ b/library/std/src/sys/sync/thread_parking/darwin.rs @@ -10,6 +10,8 @@ //! provided by libdispatch, as the underlying Mach semaphore is only dubiously //! public. +#![allow(non_camel_case_types)] + use crate::pin::Pin; use crate::sync::atomic::{ AtomicI8, diff --git a/library/std/src/sys/sync/thread_parking/futex.rs b/library/std/src/sys/sync/thread_parking/futex.rs new file mode 100644 index 00000000000..588e7b27826 --- /dev/null +++ b/library/std/src/sys/sync/thread_parking/futex.rs @@ -0,0 +1,97 @@ +use crate::pin::Pin; +use crate::sync::atomic::AtomicU32; +use crate::sync::atomic::Ordering::{Acquire, Release}; +use crate::sys::futex::{futex_wait, futex_wake}; +use crate::time::Duration; + +const PARKED: u32 = u32::MAX; +const EMPTY: u32 = 0; +const NOTIFIED: u32 = 1; + +pub struct Parker { + state: AtomicU32, +} + +// Notes about memory ordering: +// +// Memory ordering is only relevant for the relative ordering of operations +// between different variables. Even Ordering::Relaxed guarantees a +// monotonic/consistent order when looking at just a single atomic variable. +// +// So, since this parker is just a single atomic variable, we only need to look +// at the ordering guarantees we need to provide to the 'outside world'. +// +// The only memory ordering guarantee that parking and unparking provide, is +// that things which happened before unpark() are visible on the thread +// returning from park() afterwards. Otherwise, it was effectively unparked +// before unpark() was called while still consuming the 'token'. +// +// In other words, unpark() needs to synchronize with the part of park() that +// consumes the token and returns. +// +// This is done with a release-acquire synchronization, by using +// Ordering::Release when writing NOTIFIED (the 'token') in unpark(), and using +// Ordering::Acquire when checking for this state in park(). +impl Parker { + /// Construct the futex parker. The UNIX parker implementation + /// requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Self { state: AtomicU32::new(EMPTY) }); + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. + pub unsafe fn park(self: Pin<&Self>) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + loop { + // Wait for something to happen, assuming it's still set to PARKED. + futex_wait(&self.state, PARKED, None); + // Change NOTIFIED=>EMPTY and return in that case. + if self.state.compare_exchange(NOTIFIED, EMPTY, Acquire, Acquire).is_ok() { + return; + } else { + // Spurious wake up. We loop to try again. + } + } + } + + // Assumes this is only called by the thread that owns the Parker, + // which means that `self.state != PARKED`. This implementation doesn't + // require `Pin`, but other implementations do. + pub unsafe fn park_timeout(self: Pin<&Self>, timeout: Duration) { + // Change NOTIFIED=>EMPTY or EMPTY=>PARKED, and directly return in the + // first case. + if self.state.fetch_sub(1, Acquire) == NOTIFIED { + return; + } + // Wait for something to happen, assuming it's still set to PARKED. + futex_wait(&self.state, PARKED, Some(timeout)); + // This is not just a store, because we need to establish a + // release-acquire ordering with unpark(). + if self.state.swap(EMPTY, Acquire) == NOTIFIED { + // Woke up because of unpark(). + } else { + // Timeout or spurious wake up. + // We return either way, because we can't easily tell if it was the + // timeout or not. + } + } + + // This implementation doesn't require `Pin`, but other implementations do. + #[inline] + pub fn unpark(self: Pin<&Self>) { + // Change PARKED=>NOTIFIED, EMPTY=>NOTIFIED, or NOTIFIED=>NOTIFIED, and + // wake the thread in the first case. + // + // Note that even NOTIFIED=>NOTIFIED results in a write. This is on + // purpose, to make sure every unpark() has a release-acquire ordering + // with park(). + if self.state.swap(NOTIFIED, Release) == PARKED { + futex_wake(&self.state); + } + } +} diff --git a/library/std/src/sys/sync/thread_parking/id.rs b/library/std/src/sys/sync/thread_parking/id.rs new file mode 100644 index 00000000000..04667439660 --- /dev/null +++ b/library/std/src/sys/sync/thread_parking/id.rs @@ -0,0 +1,103 @@ +//! Thread parking using thread ids. +//! +//! Some platforms (notably NetBSD) have thread parking primitives whose semantics +//! match those offered by `thread::park`, with the difference that the thread to +//! be unparked is referenced by a platform-specific thread id. Since the thread +//! parker is constructed before that id is known, an atomic state variable is used +//! to manage the park state and propagate the thread id. This also avoids platform +//! calls in the case where `unpark` is called before `park`. + +use crate::cell::UnsafeCell; +use crate::pin::Pin; +use crate::sync::atomic::{ + fence, AtomicI8, + Ordering::{Acquire, Relaxed, Release}, +}; +use crate::sys::thread_parking::{current, park, park_timeout, unpark, ThreadId}; +use crate::time::Duration; + +pub struct Parker { + state: AtomicI8, + tid: UnsafeCell<Option<ThreadId>>, +} + +const PARKED: i8 = -1; +const EMPTY: i8 = 0; +const NOTIFIED: i8 = 1; + +impl Parker { + pub fn new() -> Parker { + Parker { state: AtomicI8::new(EMPTY), tid: UnsafeCell::new(None) } + } + + /// Create a new thread parker. UNIX requires this to happen in-place. + pub unsafe fn new_in_place(parker: *mut Parker) { + parker.write(Parker::new()) + } + + /// # Safety + /// * must always be called from the same thread + /// * must be called before the state is set to PARKED + unsafe fn init_tid(&self) { + // The field is only ever written to from this thread, so we don't need + // synchronization to read it here. + if self.tid.get().read().is_none() { + // Because this point is only reached once, before the state is set + // to PARKED for the first time, the non-atomic write here can not + // conflict with reads by other threads. + self.tid.get().write(Some(current())); + // Ensure that the write can be observed by all threads reading the + // state. Synchronizes with the acquire barrier in `unpark`. + fence(Release); + } + } + + pub unsafe fn park(self: Pin<&Self>) { + self.init_tid(); + + // Changes NOTIFIED to EMPTY and EMPTY to PARKED. + let state = self.state.fetch_sub(1, Acquire); + if state == EMPTY { + // Loop to guard against spurious wakeups. + // The state must be reset with acquire ordering to ensure that all + // calls to `unpark` synchronize with this thread. + while self.state.compare_exchange(NOTIFIED, EMPTY, Acquire, Relaxed).is_err() { + park(self.state.as_ptr().addr()); + } + } + } + + pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { + self.init_tid(); + + let state = self.state.fetch_sub(1, Acquire).wrapping_sub(1); + if state == PARKED { + park_timeout(dur, self.state.as_ptr().addr()); + // Swap to ensure that we observe all state changes with acquire + // ordering. + self.state.swap(EMPTY, Acquire); + } + } + + pub fn unpark(self: Pin<&Self>) { + let state = self.state.swap(NOTIFIED, Release); + if state == PARKED { + // Synchronize with the release fence in `init_tid` to observe the + // write to `tid`. + fence(Acquire); + // # Safety + // The thread id is initialized before the state is set to `PARKED` + // for the first time and is not written to from that point on + // (negating the need for an atomic read). + let tid = unsafe { self.tid.get().read().unwrap_unchecked() }; + // It is possible that the waiting thread woke up because of a timeout + // and terminated before this call is made. This call then returns an + // error or wakes up an unrelated thread. The platform API and + // environment does allow this, however. + unpark(tid, self.state.as_ptr().addr()); + } + } +} + +unsafe impl Send for Parker {} +unsafe impl Sync for Parker {} diff --git a/library/std/src/sys/sync/thread_parking/mod.rs b/library/std/src/sys/sync/thread_parking/mod.rs new file mode 100644 index 00000000000..ed1a6437faa --- /dev/null +++ b/library/std/src/sys/sync/thread_parking/mod.rs @@ -0,0 +1,37 @@ +cfg_if::cfg_if! { + if #[cfg(any( + target_os = "linux", + target_os = "android", + all(target_arch = "wasm32", target_feature = "atomics"), + target_os = "freebsd", + target_os = "openbsd", + target_os = "dragonfly", + target_os = "fuchsia", + target_os = "hermit", + ))] { + mod futex; + pub use futex::Parker; + } else if #[cfg(any( + target_os = "netbsd", + all(target_vendor = "fortanix", target_env = "sgx"), + target_os = "solid_asp3", + ))] { + mod id; + pub use id::Parker; + } else if #[cfg(target_os = "windows")] { + mod windows; + pub use windows::Parker; + } else if #[cfg(all(target_vendor = "apple", not(miri)))] { + mod darwin; + pub use darwin::Parker; + } else if #[cfg(target_os = "xous")] { + mod xous; + pub use xous::Parker; + } else if #[cfg(target_family = "unix")] { + mod pthread; + pub use pthread::Parker; + } else { + mod unsupported; + pub use unsupported::Parker; + } +} diff --git a/library/std/src/sys/pal/unix/thread_parking/pthread.rs b/library/std/src/sys/sync/thread_parking/pthread.rs index d0ad3e2ce3e..fdac1096dbf 100644 --- a/library/std/src/sys/pal/unix/thread_parking/pthread.rs +++ b/library/std/src/sys/sync/thread_parking/pthread.rs @@ -43,15 +43,7 @@ unsafe fn wait_timeout( ) { // Use the system clock on systems that do not support pthread_condattr_setclock. // This unfortunately results in problems when the system time changes. - #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - target_os = "espidf", - target_os = "horizon", - ))] + #[cfg(any(target_os = "espidf", target_os = "horizon", target_vendor = "apple"))] let (now, dur) = { use crate::cmp::min; use crate::sys::time::SystemTime; @@ -72,15 +64,7 @@ unsafe fn wait_timeout( (now, dur) }; // Use the monotonic clock on other systems. - #[cfg(not(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", - target_os = "espidf", - target_os = "horizon", - )))] + #[cfg(not(any(target_os = "espidf", target_os = "horizon", target_vendor = "apple")))] let (now, dur) = { use crate::sys::time::Timespec; @@ -122,15 +106,11 @@ impl Parker { cfg_if::cfg_if! { if #[cfg(any( - target_os = "macos", - target_os = "ios", - target_os = "tvos", - target_os = "watchos", - target_os = "visionos", target_os = "l4re", target_os = "android", target_os = "redox", target_os = "vita", + target_vendor = "apple", ))] { addr_of_mut!((*parker).cvar).write(UnsafeCell::new(libc::PTHREAD_COND_INITIALIZER)); } else if #[cfg(any(target_os = "espidf", target_os = "horizon"))] { @@ -154,7 +134,7 @@ impl Parker { // This implementation doesn't require `unsafe`, but other implementations // may assume this is only called by the thread that owns the Parker. // - // For memory ordering, see std/src/sys_common/thread_parking/futex.rs + // For memory ordering, see futex.rs pub unsafe fn park(self: Pin<&Self>) { // If we were previously notified then we consume this notification and // return quickly. diff --git a/library/std/src/sys/pal/unsupported/thread_parking.rs b/library/std/src/sys/sync/thread_parking/unsupported.rs index 197078bb186..197078bb186 100644 --- a/library/std/src/sys/pal/unsupported/thread_parking.rs +++ b/library/std/src/sys/sync/thread_parking/unsupported.rs diff --git a/library/std/src/sys/pal/windows/thread_parking.rs b/library/std/src/sys/sync/thread_parking/windows.rs index 4b8102d505a..4b8102d505a 100644 --- a/library/std/src/sys/pal/windows/thread_parking.rs +++ b/library/std/src/sys/sync/thread_parking/windows.rs diff --git a/library/std/src/sys/pal/xous/thread_parking.rs b/library/std/src/sys/sync/thread_parking/xous.rs index 0bd0462d77d..0bd0462d77d 100644 --- a/library/std/src/sys/pal/xous/thread_parking.rs +++ b/library/std/src/sys/sync/thread_parking/xous.rs diff --git a/library/std/src/sys/thread_local/fast_local.rs b/library/std/src/sys/thread_local/fast_local.rs deleted file mode 100644 index 69ee70de30c..00000000000 --- a/library/std/src/sys/thread_local/fast_local.rs +++ /dev/null @@ -1,246 +0,0 @@ -use super::lazy::LazyKeyInner; -use crate::cell::Cell; -use crate::sys::thread_local_dtor::register_dtor; -use crate::{fmt, mem, panic}; - -#[doc(hidden)] -#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] -#[allow_internal_unsafe] -#[unstable(feature = "thread_local_internals", issue = "none")] -#[rustc_macro_transparency = "semitransparent"] -pub macro thread_local_inner { - // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[inline] - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - // If the platform has support for `#[thread_local]`, use it. - #[thread_local] - // We use `UnsafeCell` here instead of `static mut` to ensure any generated TLS shims - // have a nonnull attribute on their return value. - static VAL: $crate::cell::UnsafeCell<$t> = $crate::cell::UnsafeCell::new(INIT_EXPR); - - // If a dtor isn't needed we can do something "very raw" and - // just get going. - if !$crate::mem::needs_drop::<$t>() { - unsafe { - return $crate::option::Option::Some(&*VAL.get()) - } - } - - // 0 == dtor not registered - // 1 == dtor registered, dtor not run - // 2 == dtor registered and is running or has run - #[thread_local] - static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0); - - // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires - // all that comes with it. - unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { - $crate::thread::local_impl::abort_on_dtor_unwind(|| { - let old_state = STATE.replace(2); - $crate::debug_assert_eq!(old_state, 1); - // Safety: safety requirement is passed on to caller. - unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); } - }); - } - - unsafe { - match STATE.get() { - // 0 == we haven't registered a destructor, so do - // so now. - 0 => { - $crate::thread::local_impl::Key::<$t>::register_dtor( - VAL.get() as *mut $crate::primitive::u8, - destroy, - ); - STATE.set(1); - $crate::option::Option::Some(&*VAL.get()) - } - // 1 == the destructor is registered and the value - // is valid, so return the pointer. - 1 => $crate::option::Option::Some(&*VAL.get()), - // otherwise the destructor has already run, so we - // can't give access. - _ => $crate::option::Option::None, - } - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}, - - // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - - #[inline] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - #[thread_local] - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::<$t>::new(); - - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } - if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - } - }, - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { - $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); - }, -} - -#[derive(Copy, Clone)] -enum DtorState { - Unregistered, - Registered, - RunningOrHasRun, -} - -// This data structure has been carefully constructed so that the fast path -// only contains one branch on x86. That optimization is necessary to avoid -// duplicated tls lookups on OSX. -// -// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 -pub struct Key<T> { - // If `LazyKeyInner::get` returns `None`, that indicates either: - // * The value has never been initialized - // * The value is being recursively initialized - // * The value has already been destroyed or is being destroyed - // To determine which kind of `None`, check `dtor_state`. - // - // This is very optimizer friendly for the fast path - initialized but - // not yet dropped. - inner: LazyKeyInner<T>, - - // Metadata to keep track of the state of the destructor. Remember that - // this variable is thread-local, not global. - dtor_state: Cell<DtorState>, -} - -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } -} -impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } - } - - // note that this is just a publicly-callable function only for the - // const-initialized form of thread locals, basically a way to call the - // free `register_dtor` function defined elsewhere in std. - pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - unsafe { - register_dtor(a, dtor); - } - } - - pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See the definitions of `LazyKeyInner::get` and - // `try_initialize` for more information. - // - // The caller must ensure no mutable references are ever active to - // the inner cell or the inner T when this is called. - // The `try_initialize` is dependant on the passed `init` function - // for this. - unsafe { - match self.inner.get() { - Some(val) => Some(val), - None => self.try_initialize(init), - } - } - } - - // `try_initialize` is only called once per fast thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - // - // Macos: Inlining this function can cause two `tlv_get_addr` calls to - // be performed for every call to `Key::get`. - // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 - #[inline(never)] - unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See comment above (this function doc). - if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { - // SAFETY: See comment above (this function doc). - Some(unsafe { self.inner.initialize(init) }) - } else { - None - } - } - - // `try_register_dtor` is only called once per fast thread local - // variable, except in corner cases where thread_local dtors reference - // other thread_local's, or it is being recursively initialized. - unsafe fn try_register_dtor(&self) -> bool { - match self.dtor_state.get() { - DtorState::Unregistered => { - // SAFETY: dtor registration happens before initialization. - // Passing `self` as a pointer while using `destroy_value<T>` - // is safe because the function will build a pointer to a - // Key<T>, which is the type of self and so find the correct - // size. - unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) }; - self.dtor_state.set(DtorState::Registered); - true - } - DtorState::Registered => { - // recursively initialized - true - } - DtorState::RunningOrHasRun => false, - } - } -} - -unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { - let ptr = ptr as *mut Key<T>; - - // SAFETY: - // - // The pointer `ptr` has been built just above and comes from - // `try_register_dtor` where it is originally a Key<T> coming from `self`, - // making it non-NUL and of the correct type. - // - // Right before we run the user destructor be sure to set the - // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This - // causes future calls to `get` to run `try_initialize_drop` again, - // which will now fail, and return `None`. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { - let value = (*ptr).inner.take(); - (*ptr).dtor_state.set(DtorState::RunningOrHasRun); - drop(value); - })) { - rtabort!("thread local panicked on drop"); - } -} diff --git a/library/std/src/sys/thread_local/fast_local/eager.rs b/library/std/src/sys/thread_local/fast_local/eager.rs new file mode 100644 index 00000000000..c2bc580530b --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/eager.rs @@ -0,0 +1,82 @@ +use crate::cell::{Cell, UnsafeCell}; +use crate::ptr::{self, drop_in_place}; +use crate::sys::thread_local::abort_on_dtor_unwind; +use crate::sys::thread_local_dtor::register_dtor; + +#[derive(Clone, Copy)] +enum State { + Initial, + Alive, + Destroyed, +} + +#[allow(missing_debug_implementations)] +pub struct Storage<T> { + state: Cell<State>, + val: UnsafeCell<T>, +} + +impl<T> Storage<T> { + pub const fn new(val: T) -> Storage<T> { + Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) } + } + + /// Get a reference to the TLS value. If the TLS variable has been destroyed, + /// `None` is returned. + /// + /// # Safety + /// * The `self` reference must remain valid until the TLS destructor has been + /// run. + /// * The returned reference may only be used until thread destruction occurs + /// and may not be used after reentrant initialization has occurred. + /// + // FIXME(#110897): return NonNull instead of lying about the lifetime. + #[inline] + pub unsafe fn get(&self) -> Option<&'static T> { + match self.state.get() { + // SAFETY: as the state is not `Destroyed`, the value cannot have + // been destroyed yet. The reference fulfills the terms outlined + // above. + State::Alive => unsafe { Some(&*self.val.get()) }, + State::Destroyed => None, + State::Initial => unsafe { self.initialize() }, + } + } + + #[cold] + unsafe fn initialize(&self) -> Option<&'static T> { + // Register the destructor + + // SAFETY: + // * the destructor will be called at thread destruction. + // * the caller guarantees that `self` will be valid until that time. + unsafe { + register_dtor(ptr::from_ref(self).cast_mut().cast(), destroy::<T>); + } + self.state.set(State::Alive); + // SAFETY: as the state is not `Destroyed`, the value cannot have + // been destroyed yet. The reference fulfills the terms outlined + // above. + unsafe { Some(&*self.val.get()) } + } +} + +/// Transition an `Alive` TLS variable into the `Destroyed` state, dropping its +/// value. +/// +/// # Safety +/// * Must only be called at thread destruction. +/// * `ptr` must point to an instance of `Storage` with `Alive` state and be +/// valid for accessing that instance. +unsafe extern "C" fn destroy<T>(ptr: *mut u8) { + // Print a nice abort message if a panic occurs. + abort_on_dtor_unwind(|| { + let storage = unsafe { &*(ptr as *const Storage<T>) }; + // Update the state before running the destructor as it may attempt to + // access the variable. + storage.state.set(State::Destroyed); + unsafe { + drop_in_place(storage.val.get()); + } + }) +} diff --git a/library/std/src/sys/thread_local/fast_local/lazy.rs b/library/std/src/sys/thread_local/fast_local/lazy.rs new file mode 100644 index 00000000000..c2e9a171454 --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/lazy.rs @@ -0,0 +1,121 @@ +use crate::cell::UnsafeCell; +use crate::hint::unreachable_unchecked; +use crate::ptr; +use crate::sys::thread_local::abort_on_dtor_unwind; +use crate::sys::thread_local_dtor::register_dtor; + +pub unsafe trait DestroyedState: Sized { + fn register_dtor<T>(s: &Storage<T, Self>); +} + +unsafe impl DestroyedState for ! { + fn register_dtor<T>(_: &Storage<T, !>) {} +} + +unsafe impl DestroyedState for () { + fn register_dtor<T>(s: &Storage<T, ()>) { + unsafe { + register_dtor(ptr::from_ref(s).cast_mut().cast(), destroy::<T>); + } + } +} + +enum State<T, D> { + Initial, + Alive(T), + Destroyed(D), +} + +#[allow(missing_debug_implementations)] +pub struct Storage<T, D> { + state: UnsafeCell<State<T, D>>, +} + +impl<T, D> Storage<T, D> +where + D: DestroyedState, +{ + pub const fn new() -> Storage<T, D> { + Storage { state: UnsafeCell::new(State::Initial) } + } + + /// Get a reference to the TLS value, potentially initializing it with the + /// provided parameters. If the TLS variable has been destroyed, `None` is + /// returned. + /// + /// # Safety + /// * The `self` reference must remain valid until the TLS destructor is run, + /// at which point the returned reference is invalidated. + /// * The returned reference may only be used until thread destruction occurs + /// and may not be used after reentrant initialization has occurred. + /// + // FIXME(#110897): return NonNull instead of lying about the lifetime. + #[inline] + pub unsafe fn get_or_init( + &self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // SAFETY: + // No mutable reference to the inner value exists outside the calls to + // `replace`. The lifetime of the returned reference fulfills the terms + // outlined above. + let state = unsafe { &*self.state.get() }; + match state { + State::Alive(v) => Some(v), + State::Destroyed(_) => None, + State::Initial => unsafe { self.initialize(i, f) }, + } + } + + #[cold] + unsafe fn initialize( + &self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // Perform initialization + + let v = i.and_then(Option::take).unwrap_or_else(f); + + // SAFETY: + // If references to the inner value exist, they were created in `f` + // and are invalidated here. The caller promises to never use them + // after this. + let old = unsafe { self.state.get().replace(State::Alive(v)) }; + match old { + // If the variable is not being recursively initialized, register + // the destructor. This might be a noop if the value does not need + // destruction. + State::Initial => D::register_dtor(self), + // Else, drop the old value. This might be changed to a panic. + val => drop(val), + } + + // SAFETY: + // Initialization was completed and the state was set to `Alive`, so the + // reference fulfills the terms outlined above. + unsafe { + let State::Alive(v) = &*self.state.get() else { unreachable_unchecked() }; + Some(v) + } + } +} + +/// Transition an `Alive` TLS variable into the `Destroyed` state, dropping its +/// value. +/// +/// # Safety +/// * Must only be called at thread destruction. +/// * `ptr` must point to an instance of `Storage<T, ()>` and be valid for +/// accessing that instance. +unsafe extern "C" fn destroy<T>(ptr: *mut u8) { + // Print a nice abort message if a panic occurs. + abort_on_dtor_unwind(|| { + let storage = unsafe { &*(ptr as *const Storage<T, ()>) }; + // Update the state before running the destructor as it may attempt to + // access the variable. + let val = unsafe { storage.state.get().replace(State::Destroyed(())) }; + drop(val); + }) +} diff --git a/library/std/src/sys/thread_local/fast_local/mod.rs b/library/std/src/sys/thread_local/fast_local/mod.rs new file mode 100644 index 00000000000..25379071cb7 --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/mod.rs @@ -0,0 +1,122 @@ +//! Thread local support for platforms with native TLS. +//! +//! To achieve the best performance, we choose from four different types for +//! the TLS variable, depending from the method of initialization used (`const` +//! or lazy) and the drop requirements of the stored type: +//! +//! | | `Drop` | `!Drop` | +//! |--------:|:--------------------:|:-------------------:| +//! | `const` | `EagerStorage<T>` | `T` | +//! | lazy | `LazyStorage<T, ()>` | `LazyStorage<T, !>` | +//! +//! For `const` initialization and `!Drop` types, we simply use `T` directly, +//! but for other situations, we implement a state machine to handle +//! initialization of the variable and its destructor and destruction. +//! Upon accessing the TLS variable, the current state is compared: +//! +//! 1. If the state is `Initial`, initialize the storage, transition the state +//! to `Alive` and (if applicable) register the destructor, and return a +//! reference to the value. +//! 2. If the state is `Alive`, initialization was previously completed, so +//! return a reference to the value. +//! 3. If the state is `Destroyed`, the destructor has been run already, so +//! return [`None`]. +//! +//! The TLS destructor sets the state to `Destroyed` and drops the current value. +//! +//! To simplify the code, we make `LazyStorage` generic over the destroyed state +//! and use the `!` type (never type) as type parameter for `!Drop` types. This +//! eliminates the `Destroyed` state for these values, which can allow more niche +//! optimizations to occur for the `State` enum. For `Drop` types, `()` is used. + +#![deny(unsafe_op_in_unsafe_fn)] + +mod eager; +mod lazy; + +pub use eager::Storage as EagerStorage; +pub use lazy::Storage as LazyStorage; + +#[doc(hidden)] +#[allow_internal_unstable( + thread_local_internals, + cfg_target_thread_local, + thread_local, + never_type +)] +#[allow_internal_unsafe] +#[unstable(feature = "thread_local_internals", issue = "none")] +#[rustc_macro_transparency = "semitransparent"] +pub macro thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + const __INIT: $t = $init; + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::EagerStorage; + use $crate::mem::needs_drop; + use $crate::ptr::addr_of; + + if needs_drop::<$t>() { + #[thread_local] + static VAL: EagerStorage<$t> = EagerStorage::new(__INIT); + unsafe { + VAL.get() + } + } else { + #[thread_local] + static VAL: $t = __INIT; + unsafe { + $crate::option::Option::Some(&*addr_of!(VAL)) + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, + + // used to generate the `LocalKey` value for `thread_local!` + (@key $t:ty, $init:expr) => {{ + #[inline] + fn __init() -> $t { + $init + } + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::LazyStorage; + use $crate::mem::needs_drop; + + if needs_drop::<$t>() { + #[thread_local] + static VAL: LazyStorage<$t, ()> = LazyStorage::new(); + unsafe { + VAL.get_or_init(init, __init) + } + } else { + #[thread_local] + static VAL: LazyStorage<$t, !> = LazyStorage::new(); + unsafe { + VAL.get_or_init(init, __init) + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { + $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = + $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); + }, +} diff --git a/library/std/src/sys/thread_local/mod.rs b/library/std/src/sys/thread_local/mod.rs index 8b2c839f837..0a78a1a1cf0 100644 --- a/library/std/src/sys/thread_local/mod.rs +++ b/library/std/src/sys/thread_local/mod.rs @@ -1,4 +1,5 @@ #![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] +#![cfg_attr(test, allow(unused))] // There are three thread-local implementations: "static", "fast", "OS". // The "OS" thread local key type is accessed via platform-specific API calls and is slow, while the @@ -10,12 +11,12 @@ cfg_if::cfg_if! { #[doc(hidden)] mod static_local; #[doc(hidden)] - pub use static_local::{Key, thread_local_inner}; + pub use static_local::{EagerStorage, LazyStorage, thread_local_inner}; } else if #[cfg(target_thread_local)] { #[doc(hidden)] mod fast_local; #[doc(hidden)] - pub use fast_local::{Key, thread_local_inner}; + pub use fast_local::{EagerStorage, LazyStorage, thread_local_inner}; } else { #[doc(hidden)] mod os_local; @@ -24,89 +25,12 @@ cfg_if::cfg_if! { } } -mod lazy { - use crate::cell::UnsafeCell; - use crate::hint; - use crate::mem; - - pub struct LazyKeyInner<T> { - inner: UnsafeCell<Option<T>>, - } - - impl<T> LazyKeyInner<T> { - pub const fn new() -> LazyKeyInner<T> { - LazyKeyInner { inner: UnsafeCell::new(None) } - } - - pub unsafe fn get(&self) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - unsafe { (*self.inner.get()).as_ref() } - } - - /// The caller must ensure that no reference is active: this method - /// needs unique access. - pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T { - // Execute the initialization up front, *then* move it into our slot, - // just in case initialization fails. - let value = init(); - let ptr = self.inner.get(); - - // SAFETY: - // - // note that this can in theory just be `*ptr = Some(value)`, but due to - // the compiler will currently codegen that pattern with something like: - // - // ptr::drop_in_place(ptr) - // ptr::write(ptr, Some(value)) - // - // Due to this pattern it's possible for the destructor of the value in - // `ptr` (e.g., if this is being recursively initialized) to re-access - // TLS, in which case there will be a `&` and `&mut` pointer to the same - // value (an aliasing violation). To avoid setting the "I'm running a - // destructor" flag we just use `mem::replace` which should sequence the - // operations a little differently and make this safe to call. - // - // The precondition also ensures that we are the only one accessing - // `self` at the moment so replacing is fine. - unsafe { - let _ = mem::replace(&mut *ptr, Some(value)); - } - - // SAFETY: With the call to `mem::replace` it is guaranteed there is - // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` - // will never be reached. - unsafe { - // After storing `Some` we want to get a reference to the contents of - // what we just stored. While we could use `unwrap` here and it should - // always work it empirically doesn't seem to always get optimized away, - // which means that using something like `try_with` can pull in - // panicking code and cause a large size bloat. - match *ptr { - Some(ref x) => x, - None => hint::unreachable_unchecked(), - } - } - } - - /// The other methods hand out references while taking &self. - /// As such, callers of this method must ensure no `&` and `&mut` are - /// available and used at the same time. - #[allow(unused)] - pub unsafe fn take(&mut self) -> Option<T> { - // SAFETY: See doc comment for this method. - unsafe { (*self.inner.get()).take() } - } - } -} - /// Run a callback in a scenario which must not unwind (such as a `extern "C" /// fn` declared in a user crate). If the callback unwinds anyway, then /// `rtabort` with a message about thread local panicking on drop. #[inline] -pub fn abort_on_dtor_unwind(f: impl FnOnce()) { +#[allow(dead_code)] +fn abort_on_dtor_unwind(f: impl FnOnce()) { // Using a guard like this is lower cost. let guard = DtorUnwindGuard; f(); diff --git a/library/std/src/sys/thread_local/os_local.rs b/library/std/src/sys/thread_local/os_local.rs index 3edffd7e443..d6ddbb78a9c 100644 --- a/library/std/src/sys/thread_local/os_local.rs +++ b/library/std/src/sys/thread_local/os_local.rs @@ -1,7 +1,8 @@ -use super::lazy::LazyKeyInner; +use super::abort_on_dtor_unwind; use crate::cell::Cell; -use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; -use crate::{fmt, marker, panic, ptr}; +use crate::marker::PhantomData; +use crate::ptr; +use crate::sys_common::thread_local_key::StaticKey as OsKey; #[doc(hidden)] #[allow_internal_unstable(thread_local_internals)] @@ -10,38 +11,9 @@ use crate::{fmt, marker, panic, ptr}; #[rustc_macro_transparency = "semitransparent"] pub macro thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[inline] - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // On platforms without `#[thread_local]` we fall back to the - // same implementation as below for os thread locals. - #[inline] - const fn __init() -> $t { INIT_EXPR } - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = _init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing initial value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}, + (@key $t:ty, const $init:expr) => { + $crate::thread::local_impl::thread_local_inner!(@key $t, { const INIT_EXPR: $t = $init; INIT_EXPR }) + }, // used to generate the `LocalKey` value for `thread_local!` (@key $t:ty, $init:expr) => { @@ -55,20 +27,11 @@ pub macro thread_local_inner { unsafe fn __getit( init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); + use $crate::thread::local_impl::Key; + static __KEY: Key<$t> = Key::new(); unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) + __KEY.get(init, __init) } } @@ -85,78 +48,78 @@ pub macro thread_local_inner { /// Use a regular global static to store this key; the state provided will then be /// thread-local. +#[allow(missing_debug_implementations)] pub struct Key<T> { - // OS-TLS key that we'll use to key off. - os: OsStaticKey, - marker: marker::PhantomData<Cell<T>>, -} - -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } + os: OsKey, + marker: PhantomData<Cell<T>>, } unsafe impl<T> Sync for Key<T> {} struct Value<T: 'static> { - inner: LazyKeyInner<T>, + value: T, key: &'static Key<T>, } impl<T: 'static> Key<T> { #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] pub const fn new() -> Key<T> { - Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData } + Key { os: OsKey::new(Some(destroy_value::<T>)), marker: PhantomData } } - /// It is a requirement for the caller to ensure that no mutable - /// reference is active when this method is called. - pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: See the documentation for this method. + /// Get the value associated with this key, initializating it if necessary. + /// + /// # Safety + /// * the returned reference must not be used after recursive initialization + /// or thread destruction occurs. + pub unsafe fn get( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // SAFETY: (FIXME: get should actually be safe) let ptr = unsafe { self.os.get() as *mut Value<T> }; if ptr.addr() > 1 { // SAFETY: the check ensured the pointer is safe (its destructor // is not running) + it is coming from a trusted source (self). - if let Some(ref value) = unsafe { (*ptr).inner.get() } { - return Some(value); - } + unsafe { Some(&(*ptr).value) } + } else { + // SAFETY: At this point we are sure we have no value and so + // initializing (or trying to) is safe. + unsafe { self.try_initialize(ptr, i, f) } } - // SAFETY: At this point we are sure we have no value and so - // initializing (or trying to) is safe. - unsafe { self.try_initialize(init) } } - // `try_initialize` is only called once per os thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: No mutable references are ever handed out meaning getting - // the value is ok. - let ptr = unsafe { self.os.get() as *mut Value<T> }; + unsafe fn try_initialize( + &'static self, + ptr: *mut Value<T>, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { if ptr.addr() == 1 { // destructor is running return None; } - let ptr = if ptr.is_null() { - // If the lookup returned null, we haven't initialized our own - // local copy, so do that now. - let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self })); - // SAFETY: At this point we are sure there is no value inside - // ptr so setting it will not affect anyone else. - unsafe { - self.os.set(ptr as *mut u8); - } - ptr - } else { - // recursive initialization - ptr - }; + let value = i.and_then(Option::take).unwrap_or_else(f); + let ptr = Box::into_raw(Box::new(Value { value, key: self })); + // SAFETY: (FIXME: get should actually be safe) + let old = unsafe { self.os.get() as *mut Value<T> }; + // SAFETY: `ptr` is a correct pointer that can be destroyed by the key destructor. + unsafe { + self.os.set(ptr as *mut u8); + } + if !old.is_null() { + // If the variable was recursively initialized, drop the old value. + // SAFETY: We cannot be inside a `LocalKey::with` scope, as the + // initializer has already returned and the next scope only starts + // after we return the pointer. Therefore, there can be no references + // to the old value. + drop(unsafe { Box::from_raw(old) }); + } - // SAFETY: ptr has been ensured as non-NUL just above an so can be - // dereferenced safely. - unsafe { Some((*ptr).inner.initialize(init)) } + // SAFETY: We just created this value above. + unsafe { Some(&(*ptr).value) } } } @@ -170,16 +133,11 @@ unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { // // Note that to prevent an infinite loop we reset it back to null right // before we return from the destructor ourselves. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(|| unsafe { - let ptr = Box::from_raw(ptr as *mut Value<T>); + abort_on_dtor_unwind(|| { + let ptr = unsafe { Box::from_raw(ptr as *mut Value<T>) }; let key = ptr.key; - key.os.set(ptr::without_provenance_mut(1)); + unsafe { key.os.set(ptr::without_provenance_mut(1)) }; drop(ptr); - key.os.set(ptr::null_mut()); - }) { - rtabort!("thread local panicked on drop"); - } + unsafe { key.os.set(ptr::null_mut()) }; + }); } diff --git a/library/std/src/sys/thread_local/static_local.rs b/library/std/src/sys/thread_local/static_local.rs index 206e62bb5e2..6beda2e7188 100644 --- a/library/std/src/sys/thread_local/static_local.rs +++ b/library/std/src/sys/thread_local/static_local.rs @@ -1,5 +1,7 @@ -use super::lazy::LazyKeyInner; -use crate::fmt; +//! On some targets like wasm there's no threads, so no need to generate +//! thread locals and we can instead just use plain statics! + +use crate::cell::UnsafeCell; #[doc(hidden)] #[allow_internal_unstable(thread_local_internals)] @@ -9,23 +11,17 @@ use crate::fmt; pub macro thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals (@key $t:ty, const $init:expr) => {{ - #[inline] // see comments below + const __INIT: $t = $init; + + #[inline] #[deny(unsafe_op_in_unsafe_fn)] - // FIXME: Use `SyncUnsafeCell` instead of allowing `static_mut_refs` lint - #[allow(static_mut_refs)] unsafe fn __getit( _init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // wasm without atomics maps directly to `static mut`, and dtors - // aren't implemented because thread dtors aren't really a thing - // on wasm right now - // - // FIXME(#84224) this should come after the `target_thread_local` - // block. - static mut VAL: $t = INIT_EXPR; - unsafe { $crate::option::Option::Some(&VAL) } + use $crate::thread::local_impl::EagerStorage; + + static VAL: EagerStorage<$t> = EagerStorage { value: __INIT }; + $crate::option::Option::Some(&VAL.value) } unsafe { @@ -34,74 +30,83 @@ pub macro thread_local_inner { }}, // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - #[inline] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); - - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } + (@key $t:ty, $init:expr) => {{ + #[inline] + fn __init() -> $t { $init } + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::LazyStorage; + + static VAL: LazyStorage<$t> = LazyStorage::new(); + unsafe { $crate::option::Option::Some(VAL.get(init, __init)) } } - }, + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); }, } -/// On some targets like wasm there's no threads, so no need to generate -/// thread locals and we can instead just use plain statics! - -pub struct Key<T> { - inner: LazyKeyInner<T>, +#[allow(missing_debug_implementations)] +pub struct EagerStorage<T> { + pub value: T, } -unsafe impl<T> Sync for Key<T> {} +// SAFETY: the target doesn't have threads. +unsafe impl<T> Sync for EagerStorage<T> {} -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } +#[allow(missing_debug_implementations)] +pub struct LazyStorage<T> { + value: UnsafeCell<Option<T>>, } -impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new() } +impl<T> LazyStorage<T> { + pub const fn new() -> LazyStorage<T> { + LazyStorage { value: UnsafeCell::new(None) } } - pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - let value = unsafe { - match self.inner.get() { - Some(ref value) => value, - None => self.inner.initialize(init), - } - }; - - Some(value) + /// Gets a reference to the contained value, initializing it if necessary. + /// + /// # Safety + /// The returned reference may not be used after reentrant initialization has occurred. + #[inline] + pub unsafe fn get( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> &'static T { + let value = unsafe { &*self.value.get() }; + match value { + Some(v) => v, + None => self.initialize(i, f), + } + } + + #[cold] + unsafe fn initialize( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> &'static T { + let value = i.and_then(Option::take).unwrap_or_else(f); + // Destroy the old value, after updating the TLS variable as the + // destructor might reference it. + // FIXME(#110897): maybe panic on recursive initialization. + unsafe { + self.value.get().replace(Some(value)); + } + // SAFETY: we just set this to `Some`. + unsafe { (*self.value.get()).as_ref().unwrap_unchecked() } } } + +// SAFETY: the target doesn't have threads. +unsafe impl<T> Sync for LazyStorage<T> {} |