diff options
Diffstat (limited to 'library/std/src')
22 files changed, 729 insertions, 709 deletions
diff --git a/library/std/src/fs/tests.rs b/library/std/src/fs/tests.rs index dfa05671ab0..62a268facb6 100644 --- a/library/std/src/fs/tests.rs +++ b/library/std/src/fs/tests.rs @@ -1431,7 +1431,7 @@ fn metadata_access_times() { assert_eq!(check!(a.modified()), check!(a.modified())); assert_eq!(check!(b.accessed()), check!(b.modified())); - if cfg!(target_os = "macos") || cfg!(target_os = "windows") { + if cfg!(target_vendor = "apple") || cfg!(target_os = "windows") { check!(a.created()); check!(b.created()); } diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index 949c543a264..9d6576fa841 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -213,9 +213,9 @@ //! [array]: prim@array //! [slice]: prim@slice -#![cfg_attr(not(feature = "restricted-std"), stable(feature = "rust1", since = "1.0.0"))] +#![cfg_attr(not(restricted_std), stable(feature = "rust1", since = "1.0.0"))] #![cfg_attr( - feature = "restricted-std", + restricted_std, unstable( feature = "restricted_std", issue = "none", @@ -395,7 +395,6 @@ #![feature(edition_panic)] #![feature(format_args_nl)] #![feature(get_many_mut)] -#![feature(lazy_cell)] #![feature(log_syntax)] #![feature(test)] #![feature(trace_macros)] diff --git a/library/std/src/path.rs b/library/std/src/path.rs index 79d800ff072..f4e1e2a38c1 100644 --- a/library/std/src/path.rs +++ b/library/std/src/path.rs @@ -1425,6 +1425,11 @@ impl PathBuf { /// If `extension` is the empty string, [`self.extension`] will be [`None`] /// afterwards, not `Some("")`. /// + /// # Panics + /// + /// Panics if the passed extension contains a path separator (see + /// [`is_separator`]). + /// /// # Caveats /// /// The new `extension` may contain dots and will be used in its entirety, @@ -1470,6 +1475,14 @@ impl PathBuf { } fn _set_extension(&mut self, extension: &OsStr) -> bool { + for &b in extension.as_encoded_bytes() { + if b < 128 { + if is_separator(b as char) { + panic!("extension cannot contain path separators: {:?}", extension); + } + } + } + let file_stem = match self.file_stem() { None => return false, Some(f) => f.as_encoded_bytes(), @@ -3323,7 +3336,7 @@ impl Error for StripPrefixError { /// /// # Examples /// -/// ## Posix paths +/// ## POSIX paths /// /// ``` /// # #[cfg(unix)] @@ -3369,9 +3382,12 @@ impl Error for StripPrefixError { /// ``` /// /// For verbatim paths this will simply return the path as given. For other -/// paths this is currently equivalent to calling [`GetFullPathNameW`][windows-path] -/// This may change in the future. +/// paths this is currently equivalent to calling +/// [`GetFullPathNameW`][windows-path]. +/// +/// Note that this [may change in the future][changes]. /// +/// [changes]: io#platform-specific-behavior /// [posix-semantics]: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 /// [windows-path]: https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew #[stable(feature = "absolute_path", since = "1.79.0")] diff --git a/library/std/src/path/tests.rs b/library/std/src/path/tests.rs index fde6ed4f0c0..2d8e50d1f88 100644 --- a/library/std/src/path/tests.rs +++ b/library/std/src/path/tests.rs @@ -1803,6 +1803,29 @@ fn test_windows_absolute() { assert_eq!(absolute(r"COM1").unwrap().as_os_str(), Path::new(r"\\.\COM1").as_os_str()); } +#[test] +#[should_panic = "path separator"] +fn test_extension_path_sep() { + let mut path = PathBuf::from("path/to/file"); + path.set_extension("d/../../../../../etc/passwd"); +} + +#[test] +#[should_panic = "path separator"] +#[cfg(windows)] +fn test_extension_path_sep_alternate() { + let mut path = PathBuf::from("path/to/file"); + path.set_extension("d\\test"); +} + +#[test] +#[cfg(not(windows))] +fn test_extension_path_sep_alternate() { + let mut path = PathBuf::from("path/to/file"); + path.set_extension("d\\test"); + assert_eq!(path, Path::new("path/to/file.d\\test")); +} + #[bench] #[cfg_attr(miri, ignore)] // Miri isn't fast... fn bench_path_cmp_fast_path_buf_sort(b: &mut test::Bencher) { diff --git a/library/std/src/sync/lazy_lock.rs b/library/std/src/sync/lazy_lock.rs index 27b59cfc8c2..d3bb3bfdff9 100644 --- a/library/std/src/sync/lazy_lock.rs +++ b/library/std/src/sync/lazy_lock.rs @@ -31,8 +31,6 @@ union Data<T, F> { /// Initialize static variables with `LazyLock`. /// /// ``` -/// #![feature(lazy_cell)] -/// /// use std::collections::HashMap; /// /// use std::sync::LazyLock; @@ -61,8 +59,6 @@ union Data<T, F> { /// ``` /// Initialize fields with `LazyLock`. /// ``` -/// #![feature(lazy_cell)] -/// /// use std::sync::LazyLock; /// /// #[derive(Debug)] @@ -76,8 +72,7 @@ union Data<T, F> { /// println!("{}", *data.number); /// } /// ``` - -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] pub struct LazyLock<T, F = fn() -> T> { once: Once, data: UnsafeCell<Data<T, F>>, @@ -85,8 +80,21 @@ pub struct LazyLock<T, F = fn() -> T> { impl<T, F: FnOnce() -> T> LazyLock<T, F> { /// Creates a new lazy value with the given initializing function. + /// + /// # Examples + /// + /// ``` + /// use std::sync::LazyLock; + /// + /// let hello = "Hello, World!".to_string(); + /// + /// let lazy = LazyLock::new(|| hello.to_uppercase()); + /// + /// assert_eq!(&*lazy, "HELLO, WORLD!"); + /// ``` #[inline] - #[unstable(feature = "lazy_cell", issue = "109736")] + #[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] pub const fn new(f: F) -> LazyLock<T, F> { LazyLock { once: Once::new(), data: UnsafeCell::new(Data { f: ManuallyDrop::new(f) }) } } @@ -107,7 +115,6 @@ impl<T, F: FnOnce() -> T> LazyLock<T, F> { /// # Examples /// /// ``` - /// #![feature(lazy_cell)] /// #![feature(lazy_cell_consume)] /// /// use std::sync::LazyLock; @@ -119,7 +126,7 @@ impl<T, F: FnOnce() -> T> LazyLock<T, F> { /// assert_eq!(&*lazy, "HELLO, WORLD!"); /// assert_eq!(LazyLock::into_inner(lazy).ok(), Some("HELLO, WORLD!".to_string())); /// ``` - #[unstable(feature = "lazy_cell_consume", issue = "109736")] + #[unstable(feature = "lazy_cell_consume", issue = "125623")] pub fn into_inner(mut this: Self) -> Result<T, F> { let state = this.once.state(); match state { @@ -145,8 +152,6 @@ impl<T, F: FnOnce() -> T> LazyLock<T, F> { /// # Examples /// /// ``` - /// #![feature(lazy_cell)] - /// /// use std::sync::LazyLock; /// /// let lazy = LazyLock::new(|| 92); @@ -155,7 +160,7 @@ impl<T, F: FnOnce() -> T> LazyLock<T, F> { /// assert_eq!(&*lazy, &92); /// ``` #[inline] - #[unstable(feature = "lazy_cell", issue = "109736")] + #[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] pub fn force(this: &LazyLock<T, F>) -> &T { this.once.call_once(|| { // SAFETY: `call_once` only runs this closure once, ever. @@ -191,7 +196,7 @@ impl<T, F> LazyLock<T, F> { } } -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T, F> Drop for LazyLock<T, F> { fn drop(&mut self) { match self.once.state() { @@ -204,7 +209,7 @@ impl<T, F> Drop for LazyLock<T, F> { } } -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T, F: FnOnce() -> T> Deref for LazyLock<T, F> { type Target = T; @@ -219,7 +224,7 @@ impl<T, F: FnOnce() -> T> Deref for LazyLock<T, F> { } } -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T: Default> Default for LazyLock<T> { /// Creates a new lazy value using `Default` as the initializing function. #[inline] @@ -228,7 +233,7 @@ impl<T: Default> Default for LazyLock<T> { } } -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T: fmt::Debug, F> fmt::Debug for LazyLock<T, F> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut d = f.debug_tuple("LazyLock"); @@ -242,13 +247,13 @@ impl<T: fmt::Debug, F> fmt::Debug for LazyLock<T, F> { // We never create a `&F` from a `&LazyLock<T, F>` so it is fine // to not impl `Sync` for `F`. -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] unsafe impl<T: Sync + Send, F: Send> Sync for LazyLock<T, F> {} // auto-derived `Send` impl is OK. -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T: RefUnwindSafe + UnwindSafe, F: UnwindSafe> RefUnwindSafe for LazyLock<T, F> {} -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] impl<T: UnwindSafe, F: UnwindSafe> UnwindSafe for LazyLock<T, F> {} #[cfg(test)] diff --git a/library/std/src/sync/mod.rs b/library/std/src/sync/mod.rs index e8c35bd48a7..fb7d601b094 100644 --- a/library/std/src/sync/mod.rs +++ b/library/std/src/sync/mod.rs @@ -179,7 +179,7 @@ pub use self::rwlock::{MappedRwLockReadGuard, MappedRwLockWriteGuard}; #[stable(feature = "rust1", since = "1.0.0")] pub use self::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -#[unstable(feature = "lazy_cell", issue = "109736")] +#[stable(feature = "lazy_cell", since = "CURRENT_RUSTC_VERSION")] pub use self::lazy_lock::LazyLock; #[stable(feature = "once_cell", since = "1.70.0")] pub use self::once_lock::OnceLock; diff --git a/library/std/src/sync/reentrant_lock.rs b/library/std/src/sync/reentrant_lock.rs index 80b9e0cf152..f7fe8eb1c7f 100644 --- a/library/std/src/sync/reentrant_lock.rs +++ b/library/std/src/sync/reentrant_lock.rs @@ -117,6 +117,9 @@ pub struct ReentrantLockGuard<'a, T: ?Sized + 'a> { impl<T: ?Sized> !Send for ReentrantLockGuard<'_, T> {} #[unstable(feature = "reentrant_lock", issue = "121440")] +unsafe impl<T: ?Sized + Sync> Sync for ReentrantLockGuard<'_, T> {} + +#[unstable(feature = "reentrant_lock", issue = "121440")] impl<T> ReentrantLock<T> { /// Creates a new re-entrant lock in an unlocked state ready for use. /// diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index bbd1d840e92..8f70cefc601 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -9,8 +9,6 @@ pub mod cmath; pub mod os_str; pub mod path; pub mod sync; -#[allow(dead_code)] -#[allow(unused_imports)] pub mod thread_local; // FIXME(117276): remove this, move feature implementations into individual diff --git a/library/std/src/sys/pal/unix/alloc.rs b/library/std/src/sys/pal/unix/alloc.rs index 2f908e3d0e9..eb3a57c212b 100644 --- a/library/std/src/sys/pal/unix/alloc.rs +++ b/library/std/src/sys/pal/unix/alloc.rs @@ -59,10 +59,9 @@ unsafe impl GlobalAlloc for System { } cfg_if::cfg_if! { - // We use posix_memalign wherever possible, but not all targets have that function. + // We use posix_memalign wherever possible, but some targets have very incomplete POSIX coverage + // so we need a fallback for those. if #[cfg(any( - target_os = "redox", - target_os = "espidf", target_os = "horizon", target_os = "vita", ))] { @@ -74,12 +73,11 @@ cfg_if::cfg_if! { #[inline] unsafe fn aligned_malloc(layout: &Layout) -> *mut u8 { let mut out = ptr::null_mut(); - // We prefer posix_memalign over aligned_malloc since with aligned_malloc, - // implementations are making almost arbitrary choices for which alignments are - // "supported", making it hard to use. For instance, some implementations require the - // size to be a multiple of the alignment (wasi emmalloc), while others require the - // alignment to be at least the pointer size (Illumos, macOS) -- which may or may not be - // standards-compliant, but that does not help us. + // We prefer posix_memalign over aligned_alloc since it is more widely available, and + // since with aligned_alloc, implementations are making almost arbitrary choices for + // which alignments are "supported", making it hard to use. For instance, some + // implementations require the size to be a multiple of the alignment (wasi emmalloc), + // while others require the alignment to be at least the pointer size (Illumos, macOS). // posix_memalign only has one, clear requirement: that the alignment be a multiple of // `sizeof(void*)`. Since these are all powers of 2, we can just use max. let align = layout.align().max(crate::mem::size_of::<usize>()); diff --git a/library/std/src/sys/pal/unix/args.rs b/library/std/src/sys/pal/unix/args.rs index 2a3298e8b4c..db2ec73148e 100644 --- a/library/std/src/sys/pal/unix/args.rs +++ b/library/std/src/sys/pal/unix/args.rs @@ -5,8 +5,9 @@ #![allow(dead_code)] // runtime init functions not used during testing -use crate::ffi::OsString; +use crate::ffi::{CStr, OsString}; use crate::fmt; +use crate::os::unix::ffi::OsStringExt; use crate::vec; /// One-time global initialization. @@ -16,7 +17,46 @@ pub unsafe fn init(argc: isize, argv: *const *const u8) { /// Returns the command line arguments pub fn args() -> Args { - imp::args() + let (argc, argv) = imp::argc_argv(); + + let mut vec = Vec::with_capacity(argc as usize); + + for i in 0..argc { + // SAFETY: `argv` is non-null if `argc` is positive, and it is + // guaranteed to be at least as long as `argc`, so reading from it + // should be safe. + let ptr = unsafe { argv.offset(i).read() }; + + // Some C commandline parsers (e.g. GLib and Qt) are replacing already + // handled arguments in `argv` with `NULL` and move them to the end. + // + // Since they can't directly ensure updates to `argc` as well, this + // means that `argc` might be bigger than the actual number of + // non-`NULL` pointers in `argv` at this point. + // + // To handle this we simply stop iterating at the first `NULL` + // argument. `argv` is also guaranteed to be `NULL`-terminated so any + // non-`NULL` arguments after the first `NULL` can safely be ignored. + if ptr.is_null() { + // NOTE: On Apple platforms, `-[NSProcessInfo arguments]` does not + // stop iterating here, but instead `continue`, always iterating + // up until it reached `argc`. + // + // This difference will only matter in very specific circumstances + // where `argc`/`argv` have been modified, but in unexpected ways, + // so it likely doesn't really matter which option we choose. + // See the following PR for further discussion: + // <https://github.com/rust-lang/rust/pull/125225> + break; + } + + // SAFETY: Just checked that the pointer is not NULL, and arguments + // are otherwise guaranteed to be valid C strings. + let cstr = unsafe { CStr::from_ptr(ptr) }; + vec.push(OsStringExt::from_vec(cstr.to_bytes().to_vec())); + } + + Args { iter: vec.into_iter() } } pub struct Args { @@ -75,9 +115,7 @@ impl DoubleEndedIterator for Args { target_os = "hurd", ))] mod imp { - use super::Args; - use crate::ffi::{CStr, OsString}; - use crate::os::unix::prelude::*; + use crate::ffi::c_char; use crate::ptr; use crate::sync::atomic::{AtomicIsize, AtomicPtr, Ordering}; @@ -126,162 +164,78 @@ mod imp { init_wrapper }; - pub fn args() -> Args { - Args { iter: clone().into_iter() } - } - - fn clone() -> Vec<OsString> { - unsafe { - // Load ARGC and ARGV, which hold the unmodified system-provided - // argc/argv, so we can read the pointed-to memory without atomics - // or synchronization. - // - // If either ARGC or ARGV is still zero or null, then either there - // really are no arguments, or someone is asking for `args()` - // before initialization has completed, and we return an empty - // list. - let argv = ARGV.load(Ordering::Relaxed); - let argc = if argv.is_null() { 0 } else { ARGC.load(Ordering::Relaxed) }; - let mut args = Vec::with_capacity(argc as usize); - for i in 0..argc { - let ptr = *argv.offset(i) as *const libc::c_char; - - // Some C commandline parsers (e.g. GLib and Qt) are replacing already - // handled arguments in `argv` with `NULL` and move them to the end. That - // means that `argc` might be bigger than the actual number of non-`NULL` - // pointers in `argv` at this point. - // - // To handle this we simply stop iterating at the first `NULL` argument. - // - // `argv` is also guaranteed to be `NULL`-terminated so any non-`NULL` arguments - // after the first `NULL` can safely be ignored. - if ptr.is_null() { - break; - } - - let cstr = CStr::from_ptr(ptr); - args.push(OsStringExt::from_vec(cstr.to_bytes().to_vec())); - } - - args - } + pub fn argc_argv() -> (isize, *const *const c_char) { + // Load ARGC and ARGV, which hold the unmodified system-provided + // argc/argv, so we can read the pointed-to memory without atomics or + // synchronization. + // + // If either ARGC or ARGV is still zero or null, then either there + // really are no arguments, or someone is asking for `args()` before + // initialization has completed, and we return an empty list. + let argv = ARGV.load(Ordering::Relaxed); + let argc = if argv.is_null() { 0 } else { ARGC.load(Ordering::Relaxed) }; + + // Cast from `*mut *const u8` to `*const *const c_char` + (argc, argv.cast()) } } +// Use `_NSGetArgc` and `_NSGetArgv` on Apple platforms. +// +// Even though these have underscores in their names, they've been available +// since since the first versions of both macOS and iOS, and are declared in +// the header `crt_externs.h`. +// +// NOTE: This header was added to the iOS 13.0 SDK, which has been the source +// of a great deal of confusion in the past about the availability of these +// APIs. +// +// NOTE(madsmtm): This has not strictly been verified to not cause App Store +// rejections; if this is found to be the case, the previous implementation +// of this used `[[NSProcessInfo processInfo] arguments]`. #[cfg(target_vendor = "apple")] mod imp { - use super::Args; - use crate::ffi::CStr; + use crate::ffi::{c_char, c_int}; - pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - - #[cfg(target_os = "macos")] - pub fn args() -> Args { - use crate::os::unix::prelude::*; - extern "C" { - // These functions are in crt_externs.h. - fn _NSGetArgc() -> *mut libc::c_int; - fn _NSGetArgv() -> *mut *mut *mut libc::c_char; - } - - let vec = unsafe { - let (argc, argv) = - (*_NSGetArgc() as isize, *_NSGetArgv() as *const *const libc::c_char); - (0..argc as isize) - .map(|i| { - let bytes = CStr::from_ptr(*argv.offset(i)).to_bytes().to_vec(); - OsStringExt::from_vec(bytes) - }) - .collect::<Vec<_>>() - }; - Args { iter: vec.into_iter() } + pub unsafe fn init(_argc: isize, _argv: *const *const u8) { + // No need to initialize anything in here, `libdyld.dylib` has already + // done the work for us. } - // As _NSGetArgc and _NSGetArgv aren't mentioned in iOS docs - // and use underscores in their names - they're most probably - // are considered private and therefore should be avoided. - // Here is another way to get arguments using the Objective-C - // runtime. - // - // In general it looks like: - // res = Vec::new() - // let args = [[NSProcessInfo processInfo] arguments] - // for i in (0..[args count]) - // res.push([args objectAtIndex:i]) - // res - #[cfg(not(target_os = "macos"))] - pub fn args() -> Args { - use crate::ffi::{c_char, c_void, OsString}; - use crate::mem; - use crate::str; - - type Sel = *const c_void; - type NsId = *const c_void; - type NSUInteger = usize; - + pub fn argc_argv() -> (isize, *const *const c_char) { extern "C" { - fn sel_registerName(name: *const c_char) -> Sel; - fn objc_getClass(class_name: *const c_char) -> NsId; - - // This must be transmuted to an appropriate function pointer type before being called. - fn objc_msgSend(); - } - - const MSG_SEND_PTR: unsafe extern "C" fn() = objc_msgSend; - const MSG_SEND_NO_ARGUMENTS_RETURN_PTR: unsafe extern "C" fn(NsId, Sel) -> *const c_void = - unsafe { mem::transmute(MSG_SEND_PTR) }; - const MSG_SEND_NO_ARGUMENTS_RETURN_NSUINTEGER: unsafe extern "C" fn( - NsId, - Sel, - ) -> NSUInteger = unsafe { mem::transmute(MSG_SEND_PTR) }; - const MSG_SEND_NSINTEGER_ARGUMENT_RETURN_PTR: unsafe extern "C" fn( - NsId, - Sel, - NSUInteger, - ) - -> *const c_void = unsafe { mem::transmute(MSG_SEND_PTR) }; - - let mut res = Vec::new(); - - unsafe { - let process_info_sel = sel_registerName(c"processInfo".as_ptr()); - let arguments_sel = sel_registerName(c"arguments".as_ptr()); - let count_sel = sel_registerName(c"count".as_ptr()); - let object_at_index_sel = sel_registerName(c"objectAtIndex:".as_ptr()); - let utf8string_sel = sel_registerName(c"UTF8String".as_ptr()); - - let klass = objc_getClass(c"NSProcessInfo".as_ptr()); - // `+[NSProcessInfo processInfo]` returns an object with +0 retain count, so no need to manually `retain/release`. - let info = MSG_SEND_NO_ARGUMENTS_RETURN_PTR(klass, process_info_sel); - - // `-[NSProcessInfo arguments]` returns an object with +0 retain count, so no need to manually `retain/release`. - let args = MSG_SEND_NO_ARGUMENTS_RETURN_PTR(info, arguments_sel); - - let cnt = MSG_SEND_NO_ARGUMENTS_RETURN_NSUINTEGER(args, count_sel); - for i in 0..cnt { - // `-[NSArray objectAtIndex:]` returns an object whose lifetime is tied to the array, so no need to manually `retain/release`. - let ns_string = - MSG_SEND_NSINTEGER_ARGUMENT_RETURN_PTR(args, object_at_index_sel, i); - // The lifetime of this pointer is tied to the NSString, as well as the current autorelease pool, which is why we heap-allocate the string below. - let utf_c_str: *const c_char = - MSG_SEND_NO_ARGUMENTS_RETURN_PTR(ns_string, utf8string_sel).cast(); - let bytes = CStr::from_ptr(utf_c_str).to_bytes(); - res.push(OsString::from(str::from_utf8(bytes).unwrap())) - } + // These functions are in crt_externs.h. + fn _NSGetArgc() -> *mut c_int; + fn _NSGetArgv() -> *mut *mut *mut c_char; } - Args { iter: res.into_iter() } + // SAFETY: The returned pointer points to a static initialized early + // in the program lifetime by `libdyld.dylib`, and as such is always + // valid. + // + // NOTE: Similar to `_NSGetEnviron`, there technically isn't anything + // protecting us against concurrent modifications to this, and there + // doesn't exist a lock that we can take. Instead, it is generally + // expected that it's only modified in `main` / before other code + // runs, so reading this here should be fine. + let argc = unsafe { _NSGetArgc().read() }; + // SAFETY: Same as above. + let argv = unsafe { _NSGetArgv().read() }; + + // Cast from `*mut *mut c_char` to `*const *const c_char` + (argc as isize, argv.cast()) } } #[cfg(any(target_os = "espidf", target_os = "vita"))] mod imp { - use super::Args; + use crate::ffi::c_char; + use crate::ptr; #[inline(always)] pub unsafe fn init(_argc: isize, _argv: *const *const u8) {} - pub fn args() -> Args { - Args { iter: Vec::new().into_iter() } + pub fn argc_argv() -> (isize, *const *const c_char) { + (0, ptr::null()) } } diff --git a/library/std/src/sys/pal/unix/kernel_copy.rs b/library/std/src/sys/pal/unix/kernel_copy.rs index 18acd5ecccd..cd38b7c07e2 100644 --- a/library/std/src/sys/pal/unix/kernel_copy.rs +++ b/library/std/src/sys/pal/unix/kernel_copy.rs @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> // We store the availability in a global to avoid unnecessary syscalls static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED); + let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { + NOT_PROBED => false, + UNAVAILABLE => return CopyResult::Fallback(0), + _ => true, + }; + syscall! { fn copy_file_range( fd_in: libc::c_int, @@ -571,25 +577,22 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> ) -> libc::ssize_t } - match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) { - NOT_PROBED => { - // EPERM can indicate seccomp filters or an immutable file. - // To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported - // and some other error (ENOSYS or EPERM) if it's not available - let result = unsafe { - cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) - }; - - if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) { - HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); - } else { - HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed); - return CopyResult::Fallback(0); - } + fn probe_copy_file_range_support() -> u8 { + // In some cases, we cannot determine availability from the first + // `copy_file_range` call. In this case, we probe with an invalid file + // descriptor so that the results are easily interpretable. + match unsafe { + cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0)) + .map_err(|e| e.raw_os_error()) + } { + Err(Some(EPERM | ENOSYS)) => UNAVAILABLE, + Err(Some(EBADF)) => AVAILABLE, + Ok(_) => panic!("unexpected copy_file_range probe success"), + // Treat other errors as the syscall + // being unavailable. + Err(_) => UNAVAILABLE, } - UNAVAILABLE => return CopyResult::Fallback(0), - _ => {} - }; + } let mut written = 0u64; while written < max_len { @@ -604,6 +607,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0)) }; + if !have_probed && copy_result.is_ok() { + have_probed = true; + HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed); + } + match copy_result { Ok(0) if written == 0 => { // fallback to work around several kernel bugs where copy_file_range will fail to @@ -619,7 +627,28 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) -> return match err.raw_os_error() { // when file offset + max_length > u64::MAX Some(EOVERFLOW) => CopyResult::Fallback(written), - Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => { + Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF)) + if written == 0 => + { + if !have_probed { + let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) { + // EPERM can indicate seccomp filters or an + // immutable file. To distinguish these + // cases we probe with invalid file + // descriptors which should result in EBADF + // if the syscall is supported and EPERM or + // ENOSYS if it's not available. + // + // For EOPNOTSUPP, see below. In the case of + // ENOSYS, we try to cover for faulty FUSE + // drivers. + probe_copy_file_range_support() + } else { + AVAILABLE + }; + HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed); + } + // Try fallback io::copy if either: // - Kernel version is < 4.5 (ENOSYS¹) // - Files are mounted on different fs (EXDEV) diff --git a/library/std/src/sys/pal/unix/mod.rs b/library/std/src/sys/pal/unix/mod.rs index 21f233e2262..735ed96bc7b 100644 --- a/library/std/src/sys/pal/unix/mod.rs +++ b/library/std/src/sys/pal/unix/mod.rs @@ -399,14 +399,13 @@ cfg_if::cfg_if! { // Use libumem for the (malloc-compatible) allocator #[link(name = "umem")] extern "C" {} - } else if #[cfg(target_os = "macos")] { + } else if #[cfg(target_vendor = "apple")] { + // Link to `libSystem.dylib`. + // + // Don't get confused by the presence of `System.framework`, + // it is a deprecated wrapper over the dynamic library. #[link(name = "System")] extern "C" {} - } else if #[cfg(all(target_vendor = "apple", not(target_os = "macos")))] { - #[link(name = "System")] - #[link(name = "objc")] - #[link(name = "Foundation", kind = "framework")] - extern "C" {} } else if #[cfg(target_os = "fuchsia")] { #[link(name = "zircon")] #[link(name = "fdio")] diff --git a/library/std/src/sys/pal/unix/os.rs b/library/std/src/sys/pal/unix/os.rs index 3a281525f8d..8afc49f5227 100644 --- a/library/std/src/sys/pal/unix/os.rs +++ b/library/std/src/sys/pal/unix/os.rs @@ -576,12 +576,36 @@ impl Iterator for Env { } } -#[cfg(target_os = "macos")] +// Use `_NSGetEnviron` on Apple platforms. +// +// `_NSGetEnviron` is the documented alternative (see `man environ`), and has +// been available since the first versions of both macOS and iOS. +// +// Nowadays, specifically since macOS 10.8, `environ` has been exposed through +// `libdyld.dylib`, which is linked via. `libSystem.dylib`: +// <https://github.com/apple-oss-distributions/dyld/blob/dyld-1160.6/libdyld/libdyldGlue.cpp#L913> +// +// So in the end, it likely doesn't really matter which option we use, but the +// performance cost of using `_NSGetEnviron` is extremely miniscule, and it +// might be ever so slightly more supported, so let's just use that. +// +// NOTE: The header where this is defined (`crt_externs.h`) was added to the +// iOS 13.0 SDK, which has been the source of a great deal of confusion in the +// past about the availability of this API. +// +// NOTE(madsmtm): Neither this nor using `environ` has been verified to not +// cause App Store rejections; if this is found to be the case, an alternative +// implementation of this is possible using `[NSProcessInfo environment]` +// - which internally uses `_NSGetEnviron` and a system-wide lock on the +// environment variables to protect against `setenv`, so using that might be +// desirable anyhow? Though it also means that we have to link to Foundation. +#[cfg(target_vendor = "apple")] pub unsafe fn environ() -> *mut *const *const c_char { libc::_NSGetEnviron() as *mut *const *const c_char } -#[cfg(not(target_os = "macos"))] +// Use the `environ` static which is part of POSIX. +#[cfg(not(target_vendor = "apple"))] pub unsafe fn environ() -> *mut *const *const c_char { extern "C" { static mut environ: *const *const c_char; diff --git a/library/std/src/sys/pal/unix/stack_overflow.rs b/library/std/src/sys/pal/unix/stack_overflow.rs index 26c49257ad0..2e5bd85327a 100644 --- a/library/std/src/sys/pal/unix/stack_overflow.rs +++ b/library/std/src/sys/pal/unix/stack_overflow.rs @@ -491,6 +491,14 @@ mod imp { } } +// This is intentionally not enabled on iOS/tvOS/watchOS/visionOS, as it uses +// several symbols that might lead to rejections from the App Store, namely +// `sigaction`, `sigaltstack`, `sysctlbyname`, `mmap`, `munmap` and `mprotect`. +// +// This might be overly cautious, though it is also what Swift does (and they +// usually have fewer qualms about forwards compatibility, since the runtime +// is shipped with the OS): +// <https://github.com/apple/swift/blob/swift-5.10-RELEASE/stdlib/public/runtime/CrashHandlerMacOS.cpp> #[cfg(not(any( target_os = "linux", target_os = "freebsd", diff --git a/library/std/src/sys/thread_local/fast_local.rs b/library/std/src/sys/thread_local/fast_local.rs deleted file mode 100644 index 49b51a729e4..00000000000 --- a/library/std/src/sys/thread_local/fast_local.rs +++ /dev/null @@ -1,247 +0,0 @@ -use super::lazy::LazyKeyInner; -use crate::cell::Cell; -use crate::sys::thread_local_dtor::register_dtor; -use crate::{fmt, mem, panic, ptr}; - -#[doc(hidden)] -#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)] -#[allow_internal_unsafe] -#[unstable(feature = "thread_local_internals", issue = "none")] -#[rustc_macro_transparency = "semitransparent"] -pub macro thread_local_inner { - // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[inline] - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - // If the platform has support for `#[thread_local]`, use it. - #[thread_local] - // We use `UnsafeCell` here instead of `static mut` to ensure any generated TLS shims - // have a nonnull attribute on their return value. - static VAL: $crate::cell::UnsafeCell<$t> = $crate::cell::UnsafeCell::new(INIT_EXPR); - - // If a dtor isn't needed we can do something "very raw" and - // just get going. - if !$crate::mem::needs_drop::<$t>() { - unsafe { - return $crate::option::Option::Some(&*VAL.get()) - } - } - - // 0 == dtor not registered - // 1 == dtor registered, dtor not run - // 2 == dtor registered and is running or has run - #[thread_local] - static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0); - - // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires - // all that comes with it. - unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) { - $crate::thread::local_impl::abort_on_dtor_unwind(|| { - let old_state = STATE.replace(2); - $crate::debug_assert_eq!(old_state, 1); - // Safety: safety requirement is passed on to caller. - unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); } - }); - } - - unsafe { - match STATE.get() { - // 0 == we haven't registered a destructor, so do - // so now. - 0 => { - $crate::thread::local_impl::Key::<$t>::register_dtor( - VAL.get() as *mut $crate::primitive::u8, - destroy, - ); - STATE.set(1); - $crate::option::Option::Some(&*VAL.get()) - } - // 1 == the destructor is registered and the value - // is valid, so return the pointer. - 1 => $crate::option::Option::Some(&*VAL.get()), - // otherwise the destructor has already run, so we - // can't give access. - _ => $crate::option::Option::None, - } - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}, - - // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - - #[inline] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - #[thread_local] - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::<$t>::new(); - - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } - if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - } - }, - ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { - $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = - $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); - }, -} - -#[derive(Copy, Clone)] -enum DtorState { - Unregistered, - Registered, - RunningOrHasRun, -} - -// This data structure has been carefully constructed so that the fast path -// only contains one branch on x86. That optimization is necessary to avoid -// duplicated tls lookups on OSX. -// -// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 -pub struct Key<T> { - // If `LazyKeyInner::get` returns `None`, that indicates either: - // * The value has never been initialized - // * The value is being recursively initialized - // * The value has already been destroyed or is being destroyed - // To determine which kind of `None`, check `dtor_state`. - // - // This is very optimizer friendly for the fast path - initialized but - // not yet dropped. - inner: LazyKeyInner<T>, - - // Metadata to keep track of the state of the destructor. Remember that - // this variable is thread-local, not global. - dtor_state: Cell<DtorState>, -} - -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } -} -impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) } - } - - // note that this is just a publicly-callable function only for the - // const-initialized form of thread locals, basically a way to call the - // free `register_dtor` function defined elsewhere in std. - pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { - unsafe { - register_dtor(a, dtor); - } - } - - pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See the definitions of `LazyKeyInner::get` and - // `try_initialize` for more information. - // - // The caller must ensure no mutable references are ever active to - // the inner cell or the inner T when this is called. - // The `try_initialize` is dependant on the passed `init` function - // for this. - unsafe { - match self.inner.get() { - Some(val) => Some(val), - None => self.try_initialize(init), - } - } - } - - // `try_initialize` is only called once per fast thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - // - // Macos: Inlining this function can cause two `tlv_get_addr` calls to - // be performed for every call to `Key::get`. - // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722 - #[inline(never)] - unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> { - // SAFETY: See comment above (this function doc). - if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } { - // SAFETY: See comment above (this function doc). - Some(unsafe { self.inner.initialize(init) }) - } else { - None - } - } - - // `try_register_dtor` is only called once per fast thread local - // variable, except in corner cases where thread_local dtors reference - // other thread_local's, or it is being recursively initialized. - unsafe fn try_register_dtor(&self) -> bool { - match self.dtor_state.get() { - DtorState::Unregistered => { - // SAFETY: dtor registration happens before initialization. - // Passing `self` as a pointer while using `destroy_value<T>` - // is safe because the function will build a pointer to a - // Key<T>, which is the type of self and so find the correct - // size. - unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) }; - self.dtor_state.set(DtorState::Registered); - true - } - DtorState::Registered => { - // recursively initialized - true - } - DtorState::RunningOrHasRun => false, - } - } -} - -unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) { - let ptr = ptr as *mut Key<T>; - - // SAFETY: - // - // The pointer `ptr` has been built just above and comes from - // `try_register_dtor` where it is originally a Key<T> coming from `self`, - // making it non-NUL and of the correct type. - // - // Right before we run the user destructor be sure to set the - // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This - // causes future calls to `get` to run `try_initialize_drop` again, - // which will now fail, and return `None`. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe { - let Key { inner, dtor_state } = &*ptr; - let value = inner.take(); - dtor_state.set(DtorState::RunningOrHasRun); - drop(value); - })) { - rtabort!("thread local panicked on drop"); - } -} diff --git a/library/std/src/sys/thread_local/fast_local/eager.rs b/library/std/src/sys/thread_local/fast_local/eager.rs new file mode 100644 index 00000000000..c2bc580530b --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/eager.rs @@ -0,0 +1,82 @@ +use crate::cell::{Cell, UnsafeCell}; +use crate::ptr::{self, drop_in_place}; +use crate::sys::thread_local::abort_on_dtor_unwind; +use crate::sys::thread_local_dtor::register_dtor; + +#[derive(Clone, Copy)] +enum State { + Initial, + Alive, + Destroyed, +} + +#[allow(missing_debug_implementations)] +pub struct Storage<T> { + state: Cell<State>, + val: UnsafeCell<T>, +} + +impl<T> Storage<T> { + pub const fn new(val: T) -> Storage<T> { + Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) } + } + + /// Get a reference to the TLS value. If the TLS variable has been destroyed, + /// `None` is returned. + /// + /// # Safety + /// * The `self` reference must remain valid until the TLS destructor has been + /// run. + /// * The returned reference may only be used until thread destruction occurs + /// and may not be used after reentrant initialization has occurred. + /// + // FIXME(#110897): return NonNull instead of lying about the lifetime. + #[inline] + pub unsafe fn get(&self) -> Option<&'static T> { + match self.state.get() { + // SAFETY: as the state is not `Destroyed`, the value cannot have + // been destroyed yet. The reference fulfills the terms outlined + // above. + State::Alive => unsafe { Some(&*self.val.get()) }, + State::Destroyed => None, + State::Initial => unsafe { self.initialize() }, + } + } + + #[cold] + unsafe fn initialize(&self) -> Option<&'static T> { + // Register the destructor + + // SAFETY: + // * the destructor will be called at thread destruction. + // * the caller guarantees that `self` will be valid until that time. + unsafe { + register_dtor(ptr::from_ref(self).cast_mut().cast(), destroy::<T>); + } + self.state.set(State::Alive); + // SAFETY: as the state is not `Destroyed`, the value cannot have + // been destroyed yet. The reference fulfills the terms outlined + // above. + unsafe { Some(&*self.val.get()) } + } +} + +/// Transition an `Alive` TLS variable into the `Destroyed` state, dropping its +/// value. +/// +/// # Safety +/// * Must only be called at thread destruction. +/// * `ptr` must point to an instance of `Storage` with `Alive` state and be +/// valid for accessing that instance. +unsafe extern "C" fn destroy<T>(ptr: *mut u8) { + // Print a nice abort message if a panic occurs. + abort_on_dtor_unwind(|| { + let storage = unsafe { &*(ptr as *const Storage<T>) }; + // Update the state before running the destructor as it may attempt to + // access the variable. + storage.state.set(State::Destroyed); + unsafe { + drop_in_place(storage.val.get()); + } + }) +} diff --git a/library/std/src/sys/thread_local/fast_local/lazy.rs b/library/std/src/sys/thread_local/fast_local/lazy.rs new file mode 100644 index 00000000000..c2e9a171454 --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/lazy.rs @@ -0,0 +1,121 @@ +use crate::cell::UnsafeCell; +use crate::hint::unreachable_unchecked; +use crate::ptr; +use crate::sys::thread_local::abort_on_dtor_unwind; +use crate::sys::thread_local_dtor::register_dtor; + +pub unsafe trait DestroyedState: Sized { + fn register_dtor<T>(s: &Storage<T, Self>); +} + +unsafe impl DestroyedState for ! { + fn register_dtor<T>(_: &Storage<T, !>) {} +} + +unsafe impl DestroyedState for () { + fn register_dtor<T>(s: &Storage<T, ()>) { + unsafe { + register_dtor(ptr::from_ref(s).cast_mut().cast(), destroy::<T>); + } + } +} + +enum State<T, D> { + Initial, + Alive(T), + Destroyed(D), +} + +#[allow(missing_debug_implementations)] +pub struct Storage<T, D> { + state: UnsafeCell<State<T, D>>, +} + +impl<T, D> Storage<T, D> +where + D: DestroyedState, +{ + pub const fn new() -> Storage<T, D> { + Storage { state: UnsafeCell::new(State::Initial) } + } + + /// Get a reference to the TLS value, potentially initializing it with the + /// provided parameters. If the TLS variable has been destroyed, `None` is + /// returned. + /// + /// # Safety + /// * The `self` reference must remain valid until the TLS destructor is run, + /// at which point the returned reference is invalidated. + /// * The returned reference may only be used until thread destruction occurs + /// and may not be used after reentrant initialization has occurred. + /// + // FIXME(#110897): return NonNull instead of lying about the lifetime. + #[inline] + pub unsafe fn get_or_init( + &self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // SAFETY: + // No mutable reference to the inner value exists outside the calls to + // `replace`. The lifetime of the returned reference fulfills the terms + // outlined above. + let state = unsafe { &*self.state.get() }; + match state { + State::Alive(v) => Some(v), + State::Destroyed(_) => None, + State::Initial => unsafe { self.initialize(i, f) }, + } + } + + #[cold] + unsafe fn initialize( + &self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // Perform initialization + + let v = i.and_then(Option::take).unwrap_or_else(f); + + // SAFETY: + // If references to the inner value exist, they were created in `f` + // and are invalidated here. The caller promises to never use them + // after this. + let old = unsafe { self.state.get().replace(State::Alive(v)) }; + match old { + // If the variable is not being recursively initialized, register + // the destructor. This might be a noop if the value does not need + // destruction. + State::Initial => D::register_dtor(self), + // Else, drop the old value. This might be changed to a panic. + val => drop(val), + } + + // SAFETY: + // Initialization was completed and the state was set to `Alive`, so the + // reference fulfills the terms outlined above. + unsafe { + let State::Alive(v) = &*self.state.get() else { unreachable_unchecked() }; + Some(v) + } + } +} + +/// Transition an `Alive` TLS variable into the `Destroyed` state, dropping its +/// value. +/// +/// # Safety +/// * Must only be called at thread destruction. +/// * `ptr` must point to an instance of `Storage<T, ()>` and be valid for +/// accessing that instance. +unsafe extern "C" fn destroy<T>(ptr: *mut u8) { + // Print a nice abort message if a panic occurs. + abort_on_dtor_unwind(|| { + let storage = unsafe { &*(ptr as *const Storage<T, ()>) }; + // Update the state before running the destructor as it may attempt to + // access the variable. + let val = unsafe { storage.state.get().replace(State::Destroyed(())) }; + drop(val); + }) +} diff --git a/library/std/src/sys/thread_local/fast_local/mod.rs b/library/std/src/sys/thread_local/fast_local/mod.rs new file mode 100644 index 00000000000..25379071cb7 --- /dev/null +++ b/library/std/src/sys/thread_local/fast_local/mod.rs @@ -0,0 +1,122 @@ +//! Thread local support for platforms with native TLS. +//! +//! To achieve the best performance, we choose from four different types for +//! the TLS variable, depending from the method of initialization used (`const` +//! or lazy) and the drop requirements of the stored type: +//! +//! | | `Drop` | `!Drop` | +//! |--------:|:--------------------:|:-------------------:| +//! | `const` | `EagerStorage<T>` | `T` | +//! | lazy | `LazyStorage<T, ()>` | `LazyStorage<T, !>` | +//! +//! For `const` initialization and `!Drop` types, we simply use `T` directly, +//! but for other situations, we implement a state machine to handle +//! initialization of the variable and its destructor and destruction. +//! Upon accessing the TLS variable, the current state is compared: +//! +//! 1. If the state is `Initial`, initialize the storage, transition the state +//! to `Alive` and (if applicable) register the destructor, and return a +//! reference to the value. +//! 2. If the state is `Alive`, initialization was previously completed, so +//! return a reference to the value. +//! 3. If the state is `Destroyed`, the destructor has been run already, so +//! return [`None`]. +//! +//! The TLS destructor sets the state to `Destroyed` and drops the current value. +//! +//! To simplify the code, we make `LazyStorage` generic over the destroyed state +//! and use the `!` type (never type) as type parameter for `!Drop` types. This +//! eliminates the `Destroyed` state for these values, which can allow more niche +//! optimizations to occur for the `State` enum. For `Drop` types, `()` is used. + +#![deny(unsafe_op_in_unsafe_fn)] + +mod eager; +mod lazy; + +pub use eager::Storage as EagerStorage; +pub use lazy::Storage as LazyStorage; + +#[doc(hidden)] +#[allow_internal_unstable( + thread_local_internals, + cfg_target_thread_local, + thread_local, + never_type +)] +#[allow_internal_unsafe] +#[unstable(feature = "thread_local_internals", issue = "none")] +#[rustc_macro_transparency = "semitransparent"] +pub macro thread_local_inner { + // used to generate the `LocalKey` value for const-initialized thread locals + (@key $t:ty, const $init:expr) => {{ + const __INIT: $t = $init; + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + _init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::EagerStorage; + use $crate::mem::needs_drop; + use $crate::ptr::addr_of; + + if needs_drop::<$t>() { + #[thread_local] + static VAL: EagerStorage<$t> = EagerStorage::new(__INIT); + unsafe { + VAL.get() + } + } else { + #[thread_local] + static VAL: $t = __INIT; + unsafe { + $crate::option::Option::Some(&*addr_of!(VAL)) + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, + + // used to generate the `LocalKey` value for `thread_local!` + (@key $t:ty, $init:expr) => {{ + #[inline] + fn __init() -> $t { + $init + } + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::LazyStorage; + use $crate::mem::needs_drop; + + if needs_drop::<$t>() { + #[thread_local] + static VAL: LazyStorage<$t, ()> = LazyStorage::new(); + unsafe { + VAL.get_or_init(init, __init) + } + } else { + #[thread_local] + static VAL: LazyStorage<$t, !> = LazyStorage::new(); + unsafe { + VAL.get_or_init(init, __init) + } + } + } + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { + $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = + $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); + }, +} diff --git a/library/std/src/sys/thread_local/mod.rs b/library/std/src/sys/thread_local/mod.rs index 7500c95d8b4..0a78a1a1cf0 100644 --- a/library/std/src/sys/thread_local/mod.rs +++ b/library/std/src/sys/thread_local/mod.rs @@ -1,4 +1,5 @@ #![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")] +#![cfg_attr(test, allow(unused))] // There are three thread-local implementations: "static", "fast", "OS". // The "OS" thread local key type is accessed via platform-specific API calls and is slow, while the @@ -10,12 +11,12 @@ cfg_if::cfg_if! { #[doc(hidden)] mod static_local; #[doc(hidden)] - pub use static_local::{Key, thread_local_inner}; + pub use static_local::{EagerStorage, LazyStorage, thread_local_inner}; } else if #[cfg(target_thread_local)] { #[doc(hidden)] mod fast_local; #[doc(hidden)] - pub use fast_local::{Key, thread_local_inner}; + pub use fast_local::{EagerStorage, LazyStorage, thread_local_inner}; } else { #[doc(hidden)] mod os_local; @@ -24,91 +25,12 @@ cfg_if::cfg_if! { } } -mod lazy { - use crate::cell::UnsafeCell; - use crate::hint; - use crate::mem; - - pub struct LazyKeyInner<T> { - inner: UnsafeCell<Option<T>>, - } - - impl<T> LazyKeyInner<T> { - pub const fn new() -> LazyKeyInner<T> { - LazyKeyInner { inner: UnsafeCell::new(None) } - } - - pub unsafe fn get(&self) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - unsafe { (*self.inner.get()).as_ref() } - } - - /// The caller must ensure that no reference is active: this method - /// needs unique access. - pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T { - // Execute the initialization up front, *then* move it into our slot, - // just in case initialization fails. - let value = init(); - let ptr = self.inner.get(); - - // SAFETY: - // - // note that this can in theory just be `*ptr = Some(value)`, but due to - // the compiler will currently codegen that pattern with something like: - // - // ptr::drop_in_place(ptr) - // ptr::write(ptr, Some(value)) - // - // Due to this pattern it's possible for the destructor of the value in - // `ptr` (e.g., if this is being recursively initialized) to re-access - // TLS, in which case there will be a `&` and `&mut` pointer to the same - // value (an aliasing violation). To avoid setting the "I'm running a - // destructor" flag we just use `mem::replace` which should sequence the - // operations a little differently and make this safe to call. - // - // The precondition also ensures that we are the only one accessing - // `self` at the moment so replacing is fine. - unsafe { - let _ = mem::replace(&mut *ptr, Some(value)); - } - - // SAFETY: With the call to `mem::replace` it is guaranteed there is - // a `Some` behind `ptr`, not a `None` so `unreachable_unchecked` - // will never be reached. - unsafe { - // After storing `Some` we want to get a reference to the contents of - // what we just stored. While we could use `unwrap` here and it should - // always work it empirically doesn't seem to always get optimized away, - // which means that using something like `try_with` can pull in - // panicking code and cause a large size bloat. - match *ptr { - Some(ref x) => x, - None => hint::unreachable_unchecked(), - } - } - } - - /// Watch out: unsynchronized internal mutability! - /// - /// # Safety - /// Causes UB if any reference to the value is used after this. - #[allow(unused)] - pub(crate) unsafe fn take(&self) -> Option<T> { - let mutable: *mut _ = UnsafeCell::get(&self.inner); - // SAFETY: That's the caller's problem. - unsafe { mutable.replace(None) } - } - } -} - /// Run a callback in a scenario which must not unwind (such as a `extern "C" /// fn` declared in a user crate). If the callback unwinds anyway, then /// `rtabort` with a message about thread local panicking on drop. #[inline] -pub fn abort_on_dtor_unwind(f: impl FnOnce()) { +#[allow(dead_code)] +fn abort_on_dtor_unwind(f: impl FnOnce()) { // Using a guard like this is lower cost. let guard = DtorUnwindGuard; f(); diff --git a/library/std/src/sys/thread_local/os_local.rs b/library/std/src/sys/thread_local/os_local.rs index 3edffd7e443..d6ddbb78a9c 100644 --- a/library/std/src/sys/thread_local/os_local.rs +++ b/library/std/src/sys/thread_local/os_local.rs @@ -1,7 +1,8 @@ -use super::lazy::LazyKeyInner; +use super::abort_on_dtor_unwind; use crate::cell::Cell; -use crate::sys_common::thread_local_key::StaticKey as OsStaticKey; -use crate::{fmt, marker, panic, ptr}; +use crate::marker::PhantomData; +use crate::ptr; +use crate::sys_common::thread_local_key::StaticKey as OsKey; #[doc(hidden)] #[allow_internal_unstable(thread_local_internals)] @@ -10,38 +11,9 @@ use crate::{fmt, marker, panic, ptr}; #[rustc_macro_transparency = "semitransparent"] pub macro thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals - (@key $t:ty, const $init:expr) => {{ - #[inline] - #[deny(unsafe_op_in_unsafe_fn)] - unsafe fn __getit( - _init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // On platforms without `#[thread_local]` we fall back to the - // same implementation as below for os thread locals. - #[inline] - const fn __init() -> $t { INIT_EXPR } - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = _init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing initial value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } - }}, + (@key $t:ty, const $init:expr) => { + $crate::thread::local_impl::thread_local_inner!(@key $t, { const INIT_EXPR: $t = $init; INIT_EXPR }) + }, // used to generate the `LocalKey` value for `thread_local!` (@key $t:ty, $init:expr) => { @@ -55,20 +27,11 @@ pub macro thread_local_inner { unsafe fn __getit( init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); + use $crate::thread::local_impl::Key; + static __KEY: Key<$t> = Key::new(); unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) + __KEY.get(init, __init) } } @@ -85,78 +48,78 @@ pub macro thread_local_inner { /// Use a regular global static to store this key; the state provided will then be /// thread-local. +#[allow(missing_debug_implementations)] pub struct Key<T> { - // OS-TLS key that we'll use to key off. - os: OsStaticKey, - marker: marker::PhantomData<Cell<T>>, -} - -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } + os: OsKey, + marker: PhantomData<Cell<T>>, } unsafe impl<T> Sync for Key<T> {} struct Value<T: 'static> { - inner: LazyKeyInner<T>, + value: T, key: &'static Key<T>, } impl<T: 'static> Key<T> { #[rustc_const_unstable(feature = "thread_local_internals", issue = "none")] pub const fn new() -> Key<T> { - Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData } + Key { os: OsKey::new(Some(destroy_value::<T>)), marker: PhantomData } } - /// It is a requirement for the caller to ensure that no mutable - /// reference is active when this method is called. - pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: See the documentation for this method. + /// Get the value associated with this key, initializating it if necessary. + /// + /// # Safety + /// * the returned reference must not be used after recursive initialization + /// or thread destruction occurs. + pub unsafe fn get( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { + // SAFETY: (FIXME: get should actually be safe) let ptr = unsafe { self.os.get() as *mut Value<T> }; if ptr.addr() > 1 { // SAFETY: the check ensured the pointer is safe (its destructor // is not running) + it is coming from a trusted source (self). - if let Some(ref value) = unsafe { (*ptr).inner.get() } { - return Some(value); - } + unsafe { Some(&(*ptr).value) } + } else { + // SAFETY: At this point we are sure we have no value and so + // initializing (or trying to) is safe. + unsafe { self.try_initialize(ptr, i, f) } } - // SAFETY: At this point we are sure we have no value and so - // initializing (or trying to) is safe. - unsafe { self.try_initialize(init) } } - // `try_initialize` is only called once per os thread local variable, - // except in corner cases where thread_local dtors reference other - // thread_local's, or it is being recursively initialized. - unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: No mutable references are ever handed out meaning getting - // the value is ok. - let ptr = unsafe { self.os.get() as *mut Value<T> }; + unsafe fn try_initialize( + &'static self, + ptr: *mut Value<T>, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> Option<&'static T> { if ptr.addr() == 1 { // destructor is running return None; } - let ptr = if ptr.is_null() { - // If the lookup returned null, we haven't initialized our own - // local copy, so do that now. - let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self })); - // SAFETY: At this point we are sure there is no value inside - // ptr so setting it will not affect anyone else. - unsafe { - self.os.set(ptr as *mut u8); - } - ptr - } else { - // recursive initialization - ptr - }; + let value = i.and_then(Option::take).unwrap_or_else(f); + let ptr = Box::into_raw(Box::new(Value { value, key: self })); + // SAFETY: (FIXME: get should actually be safe) + let old = unsafe { self.os.get() as *mut Value<T> }; + // SAFETY: `ptr` is a correct pointer that can be destroyed by the key destructor. + unsafe { + self.os.set(ptr as *mut u8); + } + if !old.is_null() { + // If the variable was recursively initialized, drop the old value. + // SAFETY: We cannot be inside a `LocalKey::with` scope, as the + // initializer has already returned and the next scope only starts + // after we return the pointer. Therefore, there can be no references + // to the old value. + drop(unsafe { Box::from_raw(old) }); + } - // SAFETY: ptr has been ensured as non-NUL just above an so can be - // dereferenced safely. - unsafe { Some((*ptr).inner.initialize(init)) } + // SAFETY: We just created this value above. + unsafe { Some(&(*ptr).value) } } } @@ -170,16 +133,11 @@ unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) { // // Note that to prevent an infinite loop we reset it back to null right // before we return from the destructor ourselves. - // - // Wrap the call in a catch to ensure unwinding is caught in the event - // a panic takes place in a destructor. - if let Err(_) = panic::catch_unwind(|| unsafe { - let ptr = Box::from_raw(ptr as *mut Value<T>); + abort_on_dtor_unwind(|| { + let ptr = unsafe { Box::from_raw(ptr as *mut Value<T>) }; let key = ptr.key; - key.os.set(ptr::without_provenance_mut(1)); + unsafe { key.os.set(ptr::without_provenance_mut(1)) }; drop(ptr); - key.os.set(ptr::null_mut()); - }) { - rtabort!("thread local panicked on drop"); - } + unsafe { key.os.set(ptr::null_mut()) }; + }); } diff --git a/library/std/src/sys/thread_local/static_local.rs b/library/std/src/sys/thread_local/static_local.rs index 162c3fbd97a..6beda2e7188 100644 --- a/library/std/src/sys/thread_local/static_local.rs +++ b/library/std/src/sys/thread_local/static_local.rs @@ -1,5 +1,7 @@ -use super::lazy::LazyKeyInner; -use crate::fmt; +//! On some targets like wasm there's no threads, so no need to generate +//! thread locals and we can instead just use plain statics! + +use crate::cell::UnsafeCell; #[doc(hidden)] #[allow_internal_unstable(thread_local_internals)] @@ -9,22 +11,17 @@ use crate::fmt; pub macro thread_local_inner { // used to generate the `LocalKey` value for const-initialized thread locals (@key $t:ty, const $init:expr) => {{ - #[inline] // see comments below + const __INIT: $t = $init; + + #[inline] #[deny(unsafe_op_in_unsafe_fn)] unsafe fn __getit( _init: $crate::option::Option<&mut $crate::option::Option<$t>>, ) -> $crate::option::Option<&'static $t> { - const INIT_EXPR: $t = $init; - - // wasm without atomics maps directly to `static mut`, and dtors - // aren't implemented because thread dtors aren't really a thing - // on wasm right now - // - // FIXME(#84224) this should come after the `target_thread_local` - // block. - static mut VAL: $t = INIT_EXPR; - // SAFETY: we only ever create shared references, so there's no mutable aliasing. - unsafe { $crate::option::Option::Some(&*$crate::ptr::addr_of!(VAL)) } + use $crate::thread::local_impl::EagerStorage; + + static VAL: EagerStorage<$t> = EagerStorage { value: __INIT }; + $crate::option::Option::Some(&VAL.value) } unsafe { @@ -33,74 +30,83 @@ pub macro thread_local_inner { }}, // used to generate the `LocalKey` value for `thread_local!` - (@key $t:ty, $init:expr) => { - { - #[inline] - fn __init() -> $t { $init } - #[inline] - unsafe fn __getit( - init: $crate::option::Option<&mut $crate::option::Option<$t>>, - ) -> $crate::option::Option<&'static $t> { - static __KEY: $crate::thread::local_impl::Key<$t> = - $crate::thread::local_impl::Key::new(); - - unsafe { - __KEY.get(move || { - if let $crate::option::Option::Some(init) = init { - if let $crate::option::Option::Some(value) = init.take() { - return value; - } else if $crate::cfg!(debug_assertions) { - $crate::unreachable!("missing default value"); - } - } - __init() - }) - } - } - - unsafe { - $crate::thread::LocalKey::new(__getit) - } + (@key $t:ty, $init:expr) => {{ + #[inline] + fn __init() -> $t { $init } + + #[inline] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn __getit( + init: $crate::option::Option<&mut $crate::option::Option<$t>>, + ) -> $crate::option::Option<&'static $t> { + use $crate::thread::local_impl::LazyStorage; + + static VAL: LazyStorage<$t> = LazyStorage::new(); + unsafe { $crate::option::Option::Some(VAL.get(init, __init)) } } - }, + + unsafe { + $crate::thread::LocalKey::new(__getit) + } + }}, ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => { $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> = $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*); }, } -/// On some targets like wasm there's no threads, so no need to generate -/// thread locals and we can instead just use plain statics! - -pub struct Key<T> { - inner: LazyKeyInner<T>, +#[allow(missing_debug_implementations)] +pub struct EagerStorage<T> { + pub value: T, } -unsafe impl<T> Sync for Key<T> {} +// SAFETY: the target doesn't have threads. +unsafe impl<T> Sync for EagerStorage<T> {} -impl<T> fmt::Debug for Key<T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Key").finish_non_exhaustive() - } +#[allow(missing_debug_implementations)] +pub struct LazyStorage<T> { + value: UnsafeCell<Option<T>>, } -impl<T> Key<T> { - pub const fn new() -> Key<T> { - Key { inner: LazyKeyInner::new() } +impl<T> LazyStorage<T> { + pub const fn new() -> LazyStorage<T> { + LazyStorage { value: UnsafeCell::new(None) } } - pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> { - // SAFETY: The caller must ensure no reference is ever handed out to - // the inner cell nor mutable reference to the Option<T> inside said - // cell. This make it safe to hand a reference, though the lifetime - // of 'static is itself unsafe, making the get method unsafe. - let value = unsafe { - match self.inner.get() { - Some(ref value) => value, - None => self.inner.initialize(init), - } - }; - - Some(value) + /// Gets a reference to the contained value, initializing it if necessary. + /// + /// # Safety + /// The returned reference may not be used after reentrant initialization has occurred. + #[inline] + pub unsafe fn get( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> &'static T { + let value = unsafe { &*self.value.get() }; + match value { + Some(v) => v, + None => self.initialize(i, f), + } + } + + #[cold] + unsafe fn initialize( + &'static self, + i: Option<&mut Option<T>>, + f: impl FnOnce() -> T, + ) -> &'static T { + let value = i.and_then(Option::take).unwrap_or_else(f); + // Destroy the old value, after updating the TLS variable as the + // destructor might reference it. + // FIXME(#110897): maybe panic on recursive initialization. + unsafe { + self.value.get().replace(Some(value)); + } + // SAFETY: we just set this to `Some`. + unsafe { (*self.value.get()).as_ref().unwrap_unchecked() } } } + +// SAFETY: the target doesn't have threads. +unsafe impl<T> Sync for LazyStorage<T> {} diff --git a/library/std/src/thread/mod.rs b/library/std/src/thread/mod.rs index 78bc9af6c4d..22215873933 100644 --- a/library/std/src/thread/mod.rs +++ b/library/std/src/thread/mod.rs @@ -205,7 +205,7 @@ cfg_if::cfg_if! { #[doc(hidden)] #[unstable(feature = "thread_local_internals", issue = "none")] pub mod local_impl { - pub use crate::sys::thread_local::{thread_local_inner, Key, abort_on_dtor_unwind}; + pub use crate::sys::thread_local::*; } } } |