diff options
Diffstat (limited to 'library/core')
47 files changed, 1598 insertions, 726 deletions
diff --git a/library/core/Cargo.toml b/library/core/Cargo.toml index b60826ee4e6..fe61f552a49 100644 --- a/library/core/Cargo.toml +++ b/library/core/Cargo.toml @@ -32,8 +32,6 @@ check-cfg = [ 'cfg(bootstrap)', 'cfg(no_fp_fmt_parse)', 'cfg(stdarch_intel_sde)', - # #[cfg(bootstrap)] - 'cfg(target_feature, values("vector-enhancements-1"))', # core use #[path] imports to portable-simd `core_simd` crate # and to stdarch `core_arch` crate which messes-up with Cargo list # of declared features, we therefor expect any feature cfg diff --git a/library/core/src/array/iter.rs b/library/core/src/array/iter.rs index 1edade41597..90f76d6d4c7 100644 --- a/library/core/src/array/iter.rs +++ b/library/core/src/array/iter.rs @@ -1,38 +1,35 @@ //! Defines the `IntoIter` owned iterator for arrays. use crate::intrinsics::transmute_unchecked; -use crate::iter::{self, FusedIterator, TrustedLen, TrustedRandomAccessNoCoerce}; +use crate::iter::{FusedIterator, TrustedLen, TrustedRandomAccessNoCoerce}; use crate::mem::MaybeUninit; use crate::num::NonZero; -use crate::ops::{IndexRange, Range}; +use crate::ops::{IndexRange, Range, Try}; use crate::{fmt, ptr}; +mod iter_inner; + +type InnerSized<T, const N: usize> = iter_inner::PolymorphicIter<[MaybeUninit<T>; N]>; +type InnerUnsized<T> = iter_inner::PolymorphicIter<[MaybeUninit<T>]>; + /// A by-value [array] iterator. #[stable(feature = "array_value_iter", since = "1.51.0")] #[rustc_insignificant_dtor] #[rustc_diagnostic_item = "ArrayIntoIter"] +#[derive(Clone)] pub struct IntoIter<T, const N: usize> { - /// This is the array we are iterating over. - /// - /// Elements with index `i` where `alive.start <= i < alive.end` have not - /// been yielded yet and are valid array entries. Elements with indices `i - /// < alive.start` or `i >= alive.end` have been yielded already and must - /// not be accessed anymore! Those dead elements might even be in a - /// completely uninitialized state! - /// - /// So the invariants are: - /// - `data[alive]` is alive (i.e. contains valid elements) - /// - `data[..alive.start]` and `data[alive.end..]` are dead (i.e. the - /// elements were already read and must not be touched anymore!) - data: [MaybeUninit<T>; N], + inner: InnerSized<T, N>, +} - /// The elements in `data` that have not been yielded yet. - /// - /// Invariants: - /// - `alive.end <= N` - /// - /// (And the `IndexRange` type requires `alive.start <= alive.end`.) - alive: IndexRange, +impl<T, const N: usize> IntoIter<T, N> { + #[inline] + fn unsize(&self) -> &InnerUnsized<T> { + &self.inner + } + #[inline] + fn unsize_mut(&mut self) -> &mut InnerUnsized<T> { + &mut self.inner + } } // Note: the `#[rustc_skip_during_method_dispatch(array)]` on `trait IntoIterator` @@ -53,6 +50,7 @@ impl<T, const N: usize> IntoIterator for [T; N] { /// 2021 edition -- see the [array] Editions section for more information. /// /// [array]: prim@array + #[inline] fn into_iter(self) -> Self::IntoIter { // SAFETY: The transmute here is actually safe. The docs of `MaybeUninit` // promise: @@ -68,7 +66,10 @@ impl<T, const N: usize> IntoIterator for [T; N] { // FIXME: If normal `transmute` ever gets smart enough to allow this // directly, use it instead of `transmute_unchecked`. let data: [MaybeUninit<T>; N] = unsafe { transmute_unchecked(self) }; - IntoIter { data, alive: IndexRange::zero_to(N) } + // SAFETY: The original array was entirely initialized and the the alive + // range we're passing here represents that fact. + let inner = unsafe { InnerSized::new_unchecked(IndexRange::zero_to(N), data) }; + IntoIter { inner } } } @@ -136,13 +137,16 @@ impl<T, const N: usize> IntoIter<T, N> { /// assert_eq!(r.collect::<Vec<_>>(), vec![10, 11, 12, 13, 14, 15]); /// ``` #[unstable(feature = "array_into_iter_constructors", issue = "91583")] + #[inline] pub const unsafe fn new_unchecked( buffer: [MaybeUninit<T>; N], initialized: Range<usize>, ) -> Self { // SAFETY: one of our safety conditions is that the range is canonical. let alive = unsafe { IndexRange::new_unchecked(initialized.start, initialized.end) }; - Self { data: buffer, alive } + // SAFETY: one of our safety condition is that these items are initialized. + let inner = unsafe { InnerSized::new_unchecked(alive, buffer) }; + IntoIter { inner } } /// Creates an iterator over `T` which returns no elements. @@ -198,172 +202,134 @@ impl<T, const N: usize> IntoIter<T, N> { /// assert_eq!(get_bytes(false).collect::<Vec<_>>(), vec![]); /// ``` #[unstable(feature = "array_into_iter_constructors", issue = "91583")] + #[inline] pub const fn empty() -> Self { - let buffer = [const { MaybeUninit::uninit() }; N]; - let initialized = 0..0; - - // SAFETY: We're telling it that none of the elements are initialized, - // which is trivially true. And ∀N: usize, 0 <= N. - unsafe { Self::new_unchecked(buffer, initialized) } + let inner = InnerSized::empty(); + IntoIter { inner } } /// Returns an immutable slice of all elements that have not been yielded /// yet. #[stable(feature = "array_value_iter", since = "1.51.0")] + #[inline] pub fn as_slice(&self) -> &[T] { - // SAFETY: We know that all elements within `alive` are properly initialized. - unsafe { - let slice = self.data.get_unchecked(self.alive.clone()); - slice.assume_init_ref() - } + self.unsize().as_slice() } /// Returns a mutable slice of all elements that have not been yielded yet. #[stable(feature = "array_value_iter", since = "1.51.0")] + #[inline] pub fn as_mut_slice(&mut self) -> &mut [T] { - // SAFETY: We know that all elements within `alive` are properly initialized. - unsafe { - let slice = self.data.get_unchecked_mut(self.alive.clone()); - slice.assume_init_mut() - } + self.unsize_mut().as_mut_slice() } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] impl<T, const N: usize> Iterator for IntoIter<T, N> { type Item = T; + + #[inline] fn next(&mut self) -> Option<Self::Item> { - // Get the next index from the front. - // - // Increasing `alive.start` by 1 maintains the invariant regarding - // `alive`. However, due to this change, for a short time, the alive - // zone is not `data[alive]` anymore, but `data[idx..alive.end]`. - self.alive.next().map(|idx| { - // Read the element from the array. - // SAFETY: `idx` is an index into the former "alive" region of the - // array. Reading this element means that `data[idx]` is regarded as - // dead now (i.e. do not touch). As `idx` was the start of the - // alive-zone, the alive zone is now `data[alive]` again, restoring - // all invariants. - unsafe { self.data.get_unchecked(idx).assume_init_read() } - }) + self.unsize_mut().next() } + #[inline] fn size_hint(&self) -> (usize, Option<usize>) { - let len = self.len(); - (len, Some(len)) + self.unsize().size_hint() } #[inline] - fn fold<Acc, Fold>(mut self, init: Acc, mut fold: Fold) -> Acc + fn fold<Acc, Fold>(mut self, init: Acc, fold: Fold) -> Acc where Fold: FnMut(Acc, Self::Item) -> Acc, { - let data = &mut self.data; - iter::ByRefSized(&mut self.alive).fold(init, |acc, idx| { - // SAFETY: idx is obtained by folding over the `alive` range, which implies the - // value is currently considered alive but as the range is being consumed each value - // we read here will only be read once and then considered dead. - fold(acc, unsafe { data.get_unchecked(idx).assume_init_read() }) - }) + self.unsize_mut().fold(init, fold) } + #[inline] + fn try_fold<B, F, R>(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try<Output = B>, + { + self.unsize_mut().try_fold(init, f) + } + + #[inline] fn count(self) -> usize { self.len() } + #[inline] fn last(mut self) -> Option<Self::Item> { self.next_back() } + #[inline] fn advance_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { - // This also moves the start, which marks them as conceptually "dropped", - // so if anything goes bad then our drop impl won't double-free them. - let range_to_drop = self.alive.take_prefix(n); - let remaining = n - range_to_drop.len(); - - // SAFETY: These elements are currently initialized, so it's fine to drop them. - unsafe { - let slice = self.data.get_unchecked_mut(range_to_drop); - slice.assume_init_drop(); - } - - NonZero::new(remaining).map_or(Ok(()), Err) + self.unsize_mut().advance_by(n) } #[inline] unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item { // SAFETY: The caller must provide an idx that is in bound of the remainder. - unsafe { self.data.as_ptr().add(self.alive.start()).add(idx).cast::<T>().read() } + let elem_ref = unsafe { self.as_mut_slice().get_unchecked_mut(idx) }; + // SAFETY: We only implement `TrustedRandomAccessNoCoerce` for types + // which are actually `Copy`, so cannot have multiple-drop issues. + unsafe { ptr::read(elem_ref) } } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] impl<T, const N: usize> DoubleEndedIterator for IntoIter<T, N> { + #[inline] fn next_back(&mut self) -> Option<Self::Item> { - // Get the next index from the back. - // - // Decreasing `alive.end` by 1 maintains the invariant regarding - // `alive`. However, due to this change, for a short time, the alive - // zone is not `data[alive]` anymore, but `data[alive.start..=idx]`. - self.alive.next_back().map(|idx| { - // Read the element from the array. - // SAFETY: `idx` is an index into the former "alive" region of the - // array. Reading this element means that `data[idx]` is regarded as - // dead now (i.e. do not touch). As `idx` was the end of the - // alive-zone, the alive zone is now `data[alive]` again, restoring - // all invariants. - unsafe { self.data.get_unchecked(idx).assume_init_read() } - }) + self.unsize_mut().next_back() } #[inline] - fn rfold<Acc, Fold>(mut self, init: Acc, mut rfold: Fold) -> Acc + fn rfold<Acc, Fold>(mut self, init: Acc, rfold: Fold) -> Acc where Fold: FnMut(Acc, Self::Item) -> Acc, { - let data = &mut self.data; - iter::ByRefSized(&mut self.alive).rfold(init, |acc, idx| { - // SAFETY: idx is obtained by folding over the `alive` range, which implies the - // value is currently considered alive but as the range is being consumed each value - // we read here will only be read once and then considered dead. - rfold(acc, unsafe { data.get_unchecked(idx).assume_init_read() }) - }) + self.unsize_mut().rfold(init, rfold) + } + + #[inline] + fn try_rfold<B, F, R>(&mut self, init: B, f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try<Output = B>, + { + self.unsize_mut().try_rfold(init, f) } + #[inline] fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { - // This also moves the end, which marks them as conceptually "dropped", - // so if anything goes bad then our drop impl won't double-free them. - let range_to_drop = self.alive.take_suffix(n); - let remaining = n - range_to_drop.len(); - - // SAFETY: These elements are currently initialized, so it's fine to drop them. - unsafe { - let slice = self.data.get_unchecked_mut(range_to_drop); - slice.assume_init_drop(); - } - - NonZero::new(remaining).map_or(Ok(()), Err) + self.unsize_mut().advance_back_by(n) } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] impl<T, const N: usize> Drop for IntoIter<T, N> { + #[inline] fn drop(&mut self) { - // SAFETY: This is safe: `as_mut_slice` returns exactly the sub-slice - // of elements that have not been moved out yet and that remain - // to be dropped. - unsafe { ptr::drop_in_place(self.as_mut_slice()) } + // `inner` now handles this, but it'd technically be a breaking change + // to remove this `impl`, even though it's useless. } } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] impl<T, const N: usize> ExactSizeIterator for IntoIter<T, N> { + #[inline] fn len(&self) -> usize { - self.alive.len() + self.inner.len() } + #[inline] fn is_empty(&self) -> bool { - self.alive.is_empty() + self.inner.len() == 0 } } @@ -397,31 +363,8 @@ where } #[stable(feature = "array_value_iter_impls", since = "1.40.0")] -impl<T: Clone, const N: usize> Clone for IntoIter<T, N> { - fn clone(&self) -> Self { - // Note, we don't really need to match the exact same alive range, so - // we can just clone into offset 0 regardless of where `self` is. - let mut new = - Self { data: [const { MaybeUninit::uninit() }; N], alive: IndexRange::zero_to(0) }; - - // Clone all alive elements. - for (src, dst) in iter::zip(self.as_slice(), &mut new.data) { - // Write a clone into the new array, then update its alive range. - // If cloning panics, we'll correctly drop the previous items. - dst.write(src.clone()); - // This addition cannot overflow as we're iterating a slice - new.alive = IndexRange::zero_to(new.alive.end() + 1); - } - - new - } -} - -#[stable(feature = "array_value_iter_impls", since = "1.40.0")] impl<T: fmt::Debug, const N: usize> fmt::Debug for IntoIter<T, N> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Only print the elements that were not yielded yet: we cannot - // access the yielded elements anymore. - f.debug_tuple("IntoIter").field(&self.as_slice()).finish() + self.unsize().fmt(f) } } diff --git a/library/core/src/array/iter/iter_inner.rs b/library/core/src/array/iter/iter_inner.rs new file mode 100644 index 00000000000..3c2343591f8 --- /dev/null +++ b/library/core/src/array/iter/iter_inner.rs @@ -0,0 +1,281 @@ +//! Defines the `IntoIter` owned iterator for arrays. + +use crate::mem::MaybeUninit; +use crate::num::NonZero; +use crate::ops::{IndexRange, NeverShortCircuit, Try}; +use crate::{fmt, iter}; + +#[allow(private_bounds)] +trait PartialDrop { + /// # Safety + /// `self[alive]` are all initialized before the call, + /// then are never used (without reinitializing them) after it. + unsafe fn partial_drop(&mut self, alive: IndexRange); +} +impl<T> PartialDrop for [MaybeUninit<T>] { + unsafe fn partial_drop(&mut self, alive: IndexRange) { + // SAFETY: We know that all elements within `alive` are properly initialized. + unsafe { self.get_unchecked_mut(alive).assume_init_drop() } + } +} +impl<T, const N: usize> PartialDrop for [MaybeUninit<T>; N] { + unsafe fn partial_drop(&mut self, alive: IndexRange) { + let slice: &mut [MaybeUninit<T>] = self; + // SAFETY: Initialized elements in the array are also initialized in the slice. + unsafe { slice.partial_drop(alive) } + } +} + +/// The internals of a by-value array iterator. +/// +/// The real `array::IntoIter<T, N>` stores a `PolymorphicIter<[MaybeUninit<T>, N]>` +/// which it unsizes to `PolymorphicIter<[MaybeUninit<T>]>` to iterate. +#[allow(private_bounds)] +pub(super) struct PolymorphicIter<DATA: ?Sized> +where + DATA: PartialDrop, +{ + /// The elements in `data` that have not been yielded yet. + /// + /// Invariants: + /// - `alive.end <= N` + /// + /// (And the `IndexRange` type requires `alive.start <= alive.end`.) + alive: IndexRange, + + /// This is the array we are iterating over. + /// + /// Elements with index `i` where `alive.start <= i < alive.end` have not + /// been yielded yet and are valid array entries. Elements with indices `i + /// < alive.start` or `i >= alive.end` have been yielded already and must + /// not be accessed anymore! Those dead elements might even be in a + /// completely uninitialized state! + /// + /// So the invariants are: + /// - `data[alive]` is alive (i.e. contains valid elements) + /// - `data[..alive.start]` and `data[alive.end..]` are dead (i.e. the + /// elements were already read and must not be touched anymore!) + data: DATA, +} + +#[allow(private_bounds)] +impl<DATA: ?Sized> PolymorphicIter<DATA> +where + DATA: PartialDrop, +{ + #[inline] + pub(super) const fn len(&self) -> usize { + self.alive.len() + } +} + +#[allow(private_bounds)] +impl<DATA: ?Sized> Drop for PolymorphicIter<DATA> +where + DATA: PartialDrop, +{ + #[inline] + fn drop(&mut self) { + // SAFETY: by our type invariant `self.alive` is exactly the initialized + // items, and this is drop so nothing can use the items afterwards. + unsafe { self.data.partial_drop(self.alive.clone()) } + } +} + +impl<T, const N: usize> PolymorphicIter<[MaybeUninit<T>; N]> { + #[inline] + pub(super) const fn empty() -> Self { + Self { alive: IndexRange::zero_to(0), data: [const { MaybeUninit::uninit() }; N] } + } + + /// # Safety + /// `data[alive]` are all initialized. + #[inline] + pub(super) const unsafe fn new_unchecked(alive: IndexRange, data: [MaybeUninit<T>; N]) -> Self { + Self { alive, data } + } +} + +impl<T: Clone, const N: usize> Clone for PolymorphicIter<[MaybeUninit<T>; N]> { + #[inline] + fn clone(&self) -> Self { + // Note, we don't really need to match the exact same alive range, so + // we can just clone into offset 0 regardless of where `self` is. + let mut new = Self::empty(); + + fn clone_into_new<U: Clone>( + source: &PolymorphicIter<[MaybeUninit<U>]>, + target: &mut PolymorphicIter<[MaybeUninit<U>]>, + ) { + // Clone all alive elements. + for (src, dst) in iter::zip(source.as_slice(), &mut target.data) { + // Write a clone into the new array, then update its alive range. + // If cloning panics, we'll correctly drop the previous items. + dst.write(src.clone()); + // This addition cannot overflow as we're iterating a slice, + // the length of which always fits in usize. + target.alive = IndexRange::zero_to(target.alive.end() + 1); + } + } + + clone_into_new(self, &mut new); + new + } +} + +impl<T> PolymorphicIter<[MaybeUninit<T>]> { + #[inline] + pub(super) fn as_slice(&self) -> &[T] { + // SAFETY: We know that all elements within `alive` are properly initialized. + unsafe { + let slice = self.data.get_unchecked(self.alive.clone()); + slice.assume_init_ref() + } + } + + #[inline] + pub(super) fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: We know that all elements within `alive` are properly initialized. + unsafe { + let slice = self.data.get_unchecked_mut(self.alive.clone()); + slice.assume_init_mut() + } + } +} + +impl<T: fmt::Debug> fmt::Debug for PolymorphicIter<[MaybeUninit<T>]> { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Only print the elements that were not yielded yet: we cannot + // access the yielded elements anymore. + f.debug_tuple("IntoIter").field(&self.as_slice()).finish() + } +} + +/// Iterator-equivalent methods. +/// +/// We don't implement the actual iterator traits because we want to implement +/// things like `try_fold` that require `Self: Sized` (which we're not). +impl<T> PolymorphicIter<[MaybeUninit<T>]> { + #[inline] + pub(super) fn next(&mut self) -> Option<T> { + // Get the next index from the front. + // + // Increasing `alive.start` by 1 maintains the invariant regarding + // `alive`. However, due to this change, for a short time, the alive + // zone is not `data[alive]` anymore, but `data[idx..alive.end]`. + self.alive.next().map(|idx| { + // Read the element from the array. + // SAFETY: `idx` is an index into the former "alive" region of the + // array. Reading this element means that `data[idx]` is regarded as + // dead now (i.e. do not touch). As `idx` was the start of the + // alive-zone, the alive zone is now `data[alive]` again, restoring + // all invariants. + unsafe { self.data.get_unchecked(idx).assume_init_read() } + }) + } + + #[inline] + pub(super) fn size_hint(&self) -> (usize, Option<usize>) { + let len = self.len(); + (len, Some(len)) + } + + #[inline] + pub(super) fn advance_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + // This also moves the start, which marks them as conceptually "dropped", + // so if anything goes bad then our drop impl won't double-free them. + let range_to_drop = self.alive.take_prefix(n); + let remaining = n - range_to_drop.len(); + + // SAFETY: These elements are currently initialized, so it's fine to drop them. + unsafe { + let slice = self.data.get_unchecked_mut(range_to_drop); + slice.assume_init_drop(); + } + + NonZero::new(remaining).map_or(Ok(()), Err) + } + + #[inline] + pub(super) fn fold<B>(&mut self, init: B, f: impl FnMut(B, T) -> B) -> B { + self.try_fold(init, NeverShortCircuit::wrap_mut_2(f)).0 + } + + #[inline] + pub(super) fn try_fold<B, F, R>(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, T) -> R, + R: Try<Output = B>, + { + // `alive` is an `IndexRange`, not an arbitrary iterator, so we can + // trust that its `try_fold` isn't going to do something weird like + // call the fold-er multiple times for the same index. + let data = &mut self.data; + self.alive.try_fold(init, move |accum, idx| { + // SAFETY: `idx` has been removed from the alive range, so we're not + // going to drop it (even if `f` panics) and thus its ok to give + // out ownership of that item to `f` to handle. + let elem = unsafe { data.get_unchecked(idx).assume_init_read() }; + f(accum, elem) + }) + } + + #[inline] + pub(super) fn next_back(&mut self) -> Option<T> { + // Get the next index from the back. + // + // Decreasing `alive.end` by 1 maintains the invariant regarding + // `alive`. However, due to this change, for a short time, the alive + // zone is not `data[alive]` anymore, but `data[alive.start..=idx]`. + self.alive.next_back().map(|idx| { + // Read the element from the array. + // SAFETY: `idx` is an index into the former "alive" region of the + // array. Reading this element means that `data[idx]` is regarded as + // dead now (i.e. do not touch). As `idx` was the end of the + // alive-zone, the alive zone is now `data[alive]` again, restoring + // all invariants. + unsafe { self.data.get_unchecked(idx).assume_init_read() } + }) + } + + #[inline] + pub(super) fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + // This also moves the end, which marks them as conceptually "dropped", + // so if anything goes bad then our drop impl won't double-free them. + let range_to_drop = self.alive.take_suffix(n); + let remaining = n - range_to_drop.len(); + + // SAFETY: These elements are currently initialized, so it's fine to drop them. + unsafe { + let slice = self.data.get_unchecked_mut(range_to_drop); + slice.assume_init_drop(); + } + + NonZero::new(remaining).map_or(Ok(()), Err) + } + + #[inline] + pub(super) fn rfold<B>(&mut self, init: B, f: impl FnMut(B, T) -> B) -> B { + self.try_rfold(init, NeverShortCircuit::wrap_mut_2(f)).0 + } + + #[inline] + pub(super) fn try_rfold<B, F, R>(&mut self, init: B, mut f: F) -> R + where + F: FnMut(B, T) -> R, + R: Try<Output = B>, + { + // `alive` is an `IndexRange`, not an arbitrary iterator, so we can + // trust that its `try_rfold` isn't going to do something weird like + // call the fold-er multiple times for the same index. + let data = &mut self.data; + self.alive.try_rfold(init, move |accum, idx| { + // SAFETY: `idx` has been removed from the alive range, so we're not + // going to drop it (even if `f` panics) and thus its ok to give + // out ownership of that item to `f` to handle. + let elem = unsafe { data.get_unchecked(idx).assume_init_read() }; + f(accum, elem) + }) + } +} diff --git a/library/core/src/array/mod.rs b/library/core/src/array/mod.rs index 28329bb0908..efa7bed7c8e 100644 --- a/library/core/src/array/mod.rs +++ b/library/core/src/array/mod.rs @@ -55,12 +55,16 @@ pub fn repeat<T: Clone, const N: usize>(val: T) -> [T; N] { from_trusted_iterator(repeat_n(val, N)) } -/// Creates an array of type [T; N], where each element `T` is the returned value from `cb` -/// using that element's index. +/// Creates an array where each element is produced by calling `f` with +/// that element's index while walking forward through the array. /// -/// # Arguments +/// This is essentially the same as writing +/// ```text +/// [f(0), f(1), f(2), …, f(N - 2), f(N - 1)] +/// ``` +/// and is similar to `(0..i).map(f)`, just for arrays not iterators. /// -/// * `cb`: Callback where the passed argument is the current array index. +/// If `N == 0`, this produces an empty array without ever calling `f`. /// /// # Example /// @@ -82,13 +86,30 @@ pub fn repeat<T: Clone, const N: usize>(val: T) -> [T; N] { /// // indexes are: 0 1 2 3 4 /// assert_eq!(bool_arr, [true, false, true, false, true]); /// ``` +/// +/// You can also capture things, for example to create an array full of clones +/// where you can't just use `[item; N]` because it's not `Copy`: +/// ``` +/// # // TBH `array::repeat` would be better for this, but it's not stable yet. +/// let my_string = String::from("Hello"); +/// let clones: [String; 42] = std::array::from_fn(|_| my_string.clone()); +/// assert!(clones.iter().all(|x| *x == my_string)); +/// ``` +/// +/// The array is generated in ascending index order, starting from the front +/// and going towards the back, so you can use closures with mutable state: +/// ``` +/// let mut state = 1; +/// let a = std::array::from_fn(|_| { let x = state; state *= 2; x }); +/// assert_eq!(a, [1, 2, 4, 8, 16, 32]); +/// ``` #[inline] #[stable(feature = "array_from_fn", since = "1.63.0")] -pub fn from_fn<T, const N: usize, F>(cb: F) -> [T; N] +pub fn from_fn<T, const N: usize, F>(f: F) -> [T; N] where F: FnMut(usize) -> T, { - try_from_fn(NeverShortCircuit::wrap_mut_1(cb)).0 + try_from_fn(NeverShortCircuit::wrap_mut_1(f)).0 } /// Creates an array `[T; N]` where each fallible array element `T` is returned by the `cb` call. diff --git a/library/core/src/bool.rs b/library/core/src/bool.rs index d525ab425e6..2016ece007e 100644 --- a/library/core/src/bool.rs +++ b/library/core/src/bool.rs @@ -61,52 +61,4 @@ impl bool { pub fn then<T, F: FnOnce() -> T>(self, f: F) -> Option<T> { if self { Some(f()) } else { None } } - - /// Returns either `true_val` or `false_val` depending on the value of - /// `self`, with a hint to the compiler that `self` is unlikely - /// to be correctly predicted by a CPU’s branch predictor. - /// - /// This method is functionally equivalent to - /// ```ignore (this is just for illustrative purposes) - /// fn select_unpredictable<T>(b: bool, true_val: T, false_val: T) -> T { - /// if b { true_val } else { false_val } - /// } - /// ``` - /// but might generate different assembly. In particular, on platforms with - /// a conditional move or select instruction (like `cmov` on x86 or `csel` - /// on ARM) the optimizer might use these instructions to avoid branches, - /// which can benefit performance if the branch predictor is struggling - /// with predicting `condition`, such as in an implementation of binary - /// search. - /// - /// Note however that this lowering is not guaranteed (on any platform) and - /// should not be relied upon when trying to write constant-time code. Also - /// be aware that this lowering might *decrease* performance if `condition` - /// is well-predictable. It is advisable to perform benchmarks to tell if - /// this function is useful. - /// - /// # Examples - /// - /// Distribute values evenly between two buckets: - /// ``` - /// #![feature(select_unpredictable)] - /// - /// use std::hash::BuildHasher; - /// - /// fn append<H: BuildHasher>(hasher: &H, v: i32, bucket_one: &mut Vec<i32>, bucket_two: &mut Vec<i32>) { - /// let hash = hasher.hash_one(&v); - /// let bucket = (hash % 2 == 0).select_unpredictable(bucket_one, bucket_two); - /// bucket.push(v); - /// } - /// # let hasher = std::collections::hash_map::RandomState::new(); - /// # let mut bucket_one = Vec::new(); - /// # let mut bucket_two = Vec::new(); - /// # append(&hasher, 42, &mut bucket_one, &mut bucket_two); - /// # assert_eq!(bucket_one.len() + bucket_two.len(), 1); - /// ``` - #[inline(always)] - #[unstable(feature = "select_unpredictable", issue = "133962")] - pub fn select_unpredictable<T>(self, true_val: T, false_val: T) -> T { - crate::intrinsics::select_unpredictable(self, true_val, false_val) - } } diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs index 09117e4968d..17231df731d 100644 --- a/library/core/src/cell.rs +++ b/library/core/src/cell.rs @@ -1156,7 +1156,9 @@ impl<T: ?Sized> RefCell<T> { /// Since this method borrows `RefCell` mutably, it is statically guaranteed /// that no borrows to the underlying data exist. The dynamic checks inherent /// in [`borrow_mut`] and most other methods of `RefCell` are therefore - /// unnecessary. + /// unnecessary. Note that this method does not reset the borrowing state if borrows were previously leaked + /// (e.g., via [`forget()`] on a [`Ref`] or [`RefMut`]). For that purpose, + /// consider using the unstable [`undo_leak`] method. /// /// This method can only be called if `RefCell` can be mutably borrowed, /// which in general is only the case directly after the `RefCell` has @@ -1167,6 +1169,8 @@ impl<T: ?Sized> RefCell<T> { /// Use [`borrow_mut`] to get mutable access to the underlying data then. /// /// [`borrow_mut`]: RefCell::borrow_mut() + /// [`forget()`]: mem::forget + /// [`undo_leak`]: RefCell::undo_leak() /// /// # Examples /// diff --git a/library/core/src/char/methods.rs b/library/core/src/char/methods.rs index fa584953bed..042925a352f 100644 --- a/library/core/src/char/methods.rs +++ b/library/core/src/char/methods.rs @@ -337,7 +337,7 @@ impl char { /// '1'.is_digit(1); /// ``` #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_char_classify", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_char_classify", since = "1.87.0")] #[inline] pub const fn is_digit(self, radix: u32) -> bool { self.to_digit(radix).is_some() @@ -886,7 +886,7 @@ impl char { /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] - #[rustc_const_stable(feature = "const_char_classify", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_char_classify", since = "1.87.0")] #[inline] pub const fn is_whitespace(self) -> bool { match self { @@ -1806,39 +1806,71 @@ const fn len_utf16(code: u32) -> usize { #[inline] pub const fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] { let len = len_utf8(code); - match (len, &mut *dst) { - (1, [a, ..]) => { - *a = code as u8; - } - (2, [a, b, ..]) => { - *a = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; - *b = (code & 0x3F) as u8 | TAG_CONT; - } - (3, [a, b, c, ..]) => { - *a = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; - *b = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *c = (code & 0x3F) as u8 | TAG_CONT; - } - (4, [a, b, c, d, ..]) => { - *a = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; - *b = (code >> 12 & 0x3F) as u8 | TAG_CONT; - *c = (code >> 6 & 0x3F) as u8 | TAG_CONT; - *d = (code & 0x3F) as u8 | TAG_CONT; - } - _ => { - const_panic!( - "encode_utf8: buffer does not have enough bytes to encode code point", - "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", - code: u32 = code, - len: usize = len, - dst_len: usize = dst.len(), - ) - } - }; + if dst.len() < len { + const_panic!( + "encode_utf8: buffer does not have enough bytes to encode code point", + "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}", + code: u32 = code, + len: usize = len, + dst_len: usize = dst.len(), + ); + } + + // SAFETY: `dst` is checked to be at least the length needed to encode the codepoint. + unsafe { encode_utf8_raw_unchecked(code, dst.as_mut_ptr()) }; + // SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds. unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) } } +/// Encodes a raw `u32` value as UTF-8 into the byte buffer pointed to by `dst`. +/// +/// Unlike `char::encode_utf8`, this method also handles codepoints in the surrogate range. +/// (Creating a `char` in the surrogate range is UB.) +/// The result is valid [generalized UTF-8] but not valid UTF-8. +/// +/// [generalized UTF-8]: https://simonsapin.github.io/wtf-8/#generalized-utf8 +/// +/// # Safety +/// +/// The behavior is undefined if the buffer pointed to by `dst` is not +/// large enough to hold the encoded codepoint. A buffer of length four +/// is large enough to encode any `char`. +/// +/// For a safe version of this function, see the [`encode_utf8_raw`] function. +#[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] +#[doc(hidden)] +#[inline] +pub const unsafe fn encode_utf8_raw_unchecked(code: u32, dst: *mut u8) { + let len = len_utf8(code); + // SAFETY: The caller must guarantee that the buffer pointed to by `dst` + // is at least `len` bytes long. + unsafe { + match len { + 1 => { + *dst = code as u8; + } + 2 => { + *dst = (code >> 6 & 0x1F) as u8 | TAG_TWO_B; + *dst.add(1) = (code & 0x3F) as u8 | TAG_CONT; + } + 3 => { + *dst = (code >> 12 & 0x0F) as u8 | TAG_THREE_B; + *dst.add(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *dst.add(2) = (code & 0x3F) as u8 | TAG_CONT; + } + 4 => { + *dst = (code >> 18 & 0x07) as u8 | TAG_FOUR_B; + *dst.add(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT; + *dst.add(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT; + *dst.add(3) = (code & 0x3F) as u8 | TAG_CONT; + } + // SAFETY: `char` always takes between 1 and 4 bytes to encode in UTF-8. + _ => crate::hint::unreachable_unchecked(), + } + } +} + /// Encodes a raw `u32` value as native endian UTF-16 into the provided `u16` buffer, /// and then returns the subslice of the buffer that contains the encoded character. /// diff --git a/library/core/src/char/mod.rs b/library/core/src/char/mod.rs index 088c709f1a2..5b9f0e2143f 100644 --- a/library/core/src/char/mod.rs +++ b/library/core/src/char/mod.rs @@ -38,7 +38,7 @@ pub use self::decode::{DecodeUtf16, DecodeUtf16Error}; #[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] pub use self::methods::encode_utf16_raw; // perma-unstable #[unstable(feature = "char_internals", reason = "exposed only for libstd", issue = "none")] -pub use self::methods::encode_utf8_raw; // perma-unstable +pub use self::methods::{encode_utf8_raw, encode_utf8_raw_unchecked}; // perma-unstable #[rustfmt::skip] use crate::ascii; diff --git a/library/core/src/clone.rs b/library/core/src/clone.rs index e0ac0bfc528..c237ac84cf4 100644 --- a/library/core/src/clone.rs +++ b/library/core/src/clone.rs @@ -216,7 +216,7 @@ pub macro Clone($item:item) { /// Use closures allow captured values to be automatically used. /// This is similar to have a closure that you would call `.use` over each captured value. #[unstable(feature = "ergonomic_clones", issue = "132290")] -#[cfg_attr(not(bootstrap), lang = "use_cloned")] +#[lang = "use_cloned"] pub trait UseCloned: Clone { // Empty. } @@ -427,7 +427,7 @@ pub unsafe trait CloneToUninit { /// read or dropped, because even if it was previously valid, it may have been partially /// overwritten. /// - /// The caller may wish to to take care to deallocate the allocation pointed to by `dest`, + /// The caller may wish to take care to deallocate the allocation pointed to by `dest`, /// if applicable, to avoid a memory leak (but this is not a requirement). /// /// Implementors should avoid leaking values by, upon unwinding, dropping all component values diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs index 0dc2cc72e06..c315131f413 100644 --- a/library/core/src/cmp.rs +++ b/library/core/src/cmp.rs @@ -2053,6 +2053,22 @@ mod impls { fn ge(&self, other: &&B) -> bool { PartialOrd::ge(*self, *other) } + #[inline] + fn __chaining_lt(&self, other: &&B) -> ControlFlow<bool> { + PartialOrd::__chaining_lt(*self, *other) + } + #[inline] + fn __chaining_le(&self, other: &&B) -> ControlFlow<bool> { + PartialOrd::__chaining_le(*self, *other) + } + #[inline] + fn __chaining_gt(&self, other: &&B) -> ControlFlow<bool> { + PartialOrd::__chaining_gt(*self, *other) + } + #[inline] + fn __chaining_ge(&self, other: &&B) -> ControlFlow<bool> { + PartialOrd::__chaining_ge(*self, *other) + } } #[stable(feature = "rust1", since = "1.0.0")] impl<A: ?Sized> Ord for &A @@ -2108,6 +2124,22 @@ mod impls { fn ge(&self, other: &&mut B) -> bool { PartialOrd::ge(*self, *other) } + #[inline] + fn __chaining_lt(&self, other: &&mut B) -> ControlFlow<bool> { + PartialOrd::__chaining_lt(*self, *other) + } + #[inline] + fn __chaining_le(&self, other: &&mut B) -> ControlFlow<bool> { + PartialOrd::__chaining_le(*self, *other) + } + #[inline] + fn __chaining_gt(&self, other: &&mut B) -> ControlFlow<bool> { + PartialOrd::__chaining_gt(*self, *other) + } + #[inline] + fn __chaining_ge(&self, other: &&mut B) -> ControlFlow<bool> { + PartialOrd::__chaining_ge(*self, *other) + } } #[stable(feature = "rust1", since = "1.0.0")] impl<A: ?Sized> Ord for &mut A diff --git a/library/core/src/contracts.rs b/library/core/src/contracts.rs index 8b79a3a7eba..495f84bce4b 100644 --- a/library/core/src/contracts.rs +++ b/library/core/src/contracts.rs @@ -2,19 +2,23 @@ pub use crate::macros::builtin::{contracts_ensures as ensures, contracts_requires as requires}; -/// Emitted by rustc as a desugaring of `#[ensures(PRED)] fn foo() -> R { ... [return R;] ... }` -/// into: `fn foo() { let _check = build_check_ensures(|ret| PRED) ... [return _check(R);] ... }` -/// (including the implicit return of the tail expression, if any). +/// This is an identity function used as part of the desugaring of the `#[ensures]` attribute. +/// +/// This is an existing hack to allow users to omit the type of the return value in their ensures +/// attribute. +/// +/// Ideally, rustc should be able to generate the type annotation. +/// The existing lowering logic makes it rather hard to add the explicit type annotation, +/// while the function call is fairly straight forward. #[unstable(feature = "contracts_internals", issue = "128044" /* compiler-team#759 */)] +// Similar to `contract_check_requires`, we need to use the user-facing +// `contracts` feature rather than the perma-unstable `contracts_internals`. +// Const-checking doesn't honor allow_internal_unstable logic used by contract expansion. +#[rustc_const_unstable(feature = "contracts", issue = "128044")] #[lang = "contract_build_check_ensures"] -#[track_caller] -pub fn build_check_ensures<Ret, C>(cond: C) -> impl (Fn(Ret) -> Ret) + Copy +pub const fn build_check_ensures<Ret, C>(cond: C) -> C where - C: for<'a> Fn(&'a Ret) -> bool + Copy + 'static, + C: Fn(&Ret) -> bool + Copy + 'static, { - #[track_caller] - move |ret| { - crate::intrinsics::contract_check_ensures(&ret, cond); - ret - } + cond } diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index ec7c1705fb8..580f95eddce 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -7,7 +7,7 @@ use crate::char::{EscapeDebugExtArgs, MAX_LEN_UTF8}; use crate::marker::PhantomData; use crate::num::fmt as numfmt; use crate::ops::Deref; -use crate::{iter, mem, result, str}; +use crate::{iter, result, str}; mod builders; #[cfg(not(no_fp_fmt_parse))] @@ -1515,19 +1515,6 @@ unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::Placeholder, args: &[rt::Argume // which guarantees the indexes are always within bounds. unsafe { (getcount(args, &arg.width), getcount(args, &arg.precision)) }; - #[cfg(bootstrap)] - let options = - *FormattingOptions { flags: flags::ALWAYS_SET | arg.flags << 21, width: 0, precision: 0 } - .align(match arg.align { - rt::Alignment::Left => Some(Alignment::Left), - rt::Alignment::Right => Some(Alignment::Right), - rt::Alignment::Center => Some(Alignment::Center), - rt::Alignment::Unknown => None, - }) - .fill(arg.fill) - .width(width) - .precision(precision); - #[cfg(not(bootstrap))] let options = FormattingOptions { flags: arg.flags, width, precision }; // Extract the correct argument @@ -1544,21 +1531,6 @@ unsafe fn run(fmt: &mut Formatter<'_>, arg: &rt::Placeholder, args: &[rt::Argume unsafe { value.fmt(fmt) } } -#[cfg(bootstrap)] -unsafe fn getcount(args: &[rt::Argument<'_>], cnt: &rt::Count) -> Option<u16> { - match *cnt { - rt::Count::Is(n) => Some(n as u16), - rt::Count::Implied => None, - rt::Count::Param(i) => { - debug_assert!(i < args.len()); - // SAFETY: cnt and args come from the same Arguments, - // which guarantees this index is always within bounds. - unsafe { args.get_unchecked(i).as_u16() } - } - } -} - -#[cfg(not(bootstrap))] unsafe fn getcount(args: &[rt::Argument<'_>], cnt: &rt::Count) -> u16 { match *cnt { rt::Count::Is(n) => n, diff --git a/library/core/src/fmt/rt.rs b/library/core/src/fmt/rt.rs index 0b04ebccae2..adcfdd309b7 100644 --- a/library/core/src/fmt/rt.rs +++ b/library/core/src/fmt/rt.rs @@ -11,10 +11,6 @@ use crate::ptr::NonNull; #[derive(Copy, Clone)] pub struct Placeholder { pub position: usize, - #[cfg(bootstrap)] - pub fill: char, - #[cfg(bootstrap)] - pub align: Alignment, pub flags: u32, pub precision: Count, pub width: Count, @@ -23,38 +19,17 @@ pub struct Placeholder { #[cfg(bootstrap)] impl Placeholder { #[inline] - pub const fn new( - position: usize, - fill: char, - align: Alignment, - flags: u32, - precision: Count, - width: Count, - ) -> Self { - Self { position, fill, align, flags, precision, width } + pub const fn new(position: usize, flags: u32, precision: Count, width: Count) -> Self { + Self { position, flags, precision, width } } } -#[cfg(bootstrap)] -#[lang = "format_alignment"] -#[derive(Copy, Clone, PartialEq, Eq)] -pub enum Alignment { - Left, - Right, - Center, - Unknown, -} - /// Used by [width](https://doc.rust-lang.org/std/fmt/#width) /// and [precision](https://doc.rust-lang.org/std/fmt/#precision) specifiers. #[lang = "format_count"] #[derive(Copy, Clone)] pub enum Count { /// Specified with a literal number, stores the value - #[cfg(bootstrap)] - Is(usize), - /// Specified with a literal number, stores the value - #[cfg(not(bootstrap))] Is(u16), /// Specified using `$` and `*` syntaxes, stores the index into `args` Param(usize), @@ -90,61 +65,92 @@ pub struct Argument<'a> { ty: ArgumentType<'a>, } -#[rustc_diagnostic_item = "ArgumentMethods"] -impl Argument<'_> { - #[inline] - const fn new<'a, T>(x: &'a T, f: fn(&T, &mut Formatter<'_>) -> Result) -> Argument<'a> { +macro_rules! argument_new { + ($t:ty, $x:expr, $f:expr) => { Argument { // INVARIANT: this creates an `ArgumentType<'a>` from a `&'a T` and // a `fn(&T, ...)`, so the invariant is maintained. ty: ArgumentType::Placeholder { - value: NonNull::from_ref(x).cast(), - // SAFETY: function pointers always have the same layout. - formatter: unsafe { mem::transmute(f) }, + value: NonNull::<$t>::from_ref($x).cast(), + // The Rust ABI considers all pointers to be equivalent, so transmuting a fn(&T) to + // fn(NonNull<()>) and calling it with a NonNull<()> that points at a T is allowed. + // However, the CFI sanitizer does not allow this, and triggers a crash when it + // happens. + // + // To avoid this crash, we use a helper function when CFI is enabled. To avoid the + // cost of this helper function (mainly code-size) when it is not needed, we + // transmute the function pointer otherwise. + // + // This is similar to what the Rust compiler does internally with vtables when KCFI + // is enabled, where it generates trampoline functions that only serve to adjust the + // expected type of the argument. `ArgumentType::Placeholder` is a bit like a + // manually constructed trait object, so it is not surprising that the same approach + // has to be applied here as well. + // + // It is still considered problematic (from the Rust side) that CFI rejects entirely + // legal Rust programs, so we do not consider anything done here a stable guarantee, + // but meanwhile we carry this work-around to keep Rust compatible with CFI and + // KCFI. + #[cfg(not(any(sanitize = "cfi", sanitize = "kcfi")))] + formatter: { + let f: fn(&$t, &mut Formatter<'_>) -> Result = $f; + // SAFETY: This is only called with `value`, which has the right type. + unsafe { core::mem::transmute(f) } + }, + #[cfg(any(sanitize = "cfi", sanitize = "kcfi"))] + formatter: |ptr: NonNull<()>, fmt: &mut Formatter<'_>| { + let func = $f; + // SAFETY: This is the same type as the `value` field. + let r = unsafe { ptr.cast::<$t>().as_ref() }; + (func)(r, fmt) + }, _lifetime: PhantomData, }, } - } + }; +} +#[rustc_diagnostic_item = "ArgumentMethods"] +impl Argument<'_> { #[inline] pub fn new_display<T: Display>(x: &T) -> Argument<'_> { - Self::new(x, Display::fmt) + argument_new!(T, x, <T as Display>::fmt) } #[inline] pub fn new_debug<T: Debug>(x: &T) -> Argument<'_> { - Self::new(x, Debug::fmt) + argument_new!(T, x, <T as Debug>::fmt) } #[inline] pub fn new_debug_noop<T: Debug>(x: &T) -> Argument<'_> { - Self::new(x, |_, _| Ok(())) + argument_new!(T, x, |_: &T, _| Ok(())) } #[inline] pub fn new_octal<T: Octal>(x: &T) -> Argument<'_> { - Self::new(x, Octal::fmt) + argument_new!(T, x, <T as Octal>::fmt) } #[inline] pub fn new_lower_hex<T: LowerHex>(x: &T) -> Argument<'_> { - Self::new(x, LowerHex::fmt) + argument_new!(T, x, <T as LowerHex>::fmt) } #[inline] pub fn new_upper_hex<T: UpperHex>(x: &T) -> Argument<'_> { - Self::new(x, UpperHex::fmt) + argument_new!(T, x, <T as UpperHex>::fmt) } #[inline] pub fn new_pointer<T: Pointer>(x: &T) -> Argument<'_> { - Self::new(x, Pointer::fmt) + argument_new!(T, x, <T as Pointer>::fmt) } #[inline] pub fn new_binary<T: Binary>(x: &T) -> Argument<'_> { - Self::new(x, Binary::fmt) + argument_new!(T, x, <T as Binary>::fmt) } #[inline] pub fn new_lower_exp<T: LowerExp>(x: &T) -> Argument<'_> { - Self::new(x, LowerExp::fmt) + argument_new!(T, x, <T as LowerExp>::fmt) } #[inline] pub fn new_upper_exp<T: UpperExp>(x: &T) -> Argument<'_> { - Self::new(x, UpperExp::fmt) + argument_new!(T, x, <T as UpperExp>::fmt) } #[inline] #[track_caller] @@ -160,11 +166,6 @@ impl Argument<'_> { /// # Safety /// /// This argument must actually be a placeholder argument. - /// - // FIXME: Transmuting formatter in new and indirectly branching to/calling - // it here is an explicit CFI violation. - #[allow(inline_no_sanitize)] - #[no_sanitize(cfi, kcfi)] #[inline] pub(super) unsafe fn fmt(&self, f: &mut Formatter<'_>) -> Result { match self.ty { diff --git a/library/core/src/hint.rs b/library/core/src/hint.rs index 5ce282b05de..1ca23ab6eea 100644 --- a/library/core/src/hint.rs +++ b/library/core/src/hint.rs @@ -4,6 +4,7 @@ //! //! Hints may be compile time or runtime. +use crate::mem::MaybeUninit; use crate::{intrinsics, ub_checks}; /// Informs the compiler that the site which is calling this function is not @@ -734,3 +735,63 @@ pub const fn unlikely(b: bool) -> bool { pub const fn cold_path() { crate::intrinsics::cold_path() } + +/// Returns either `true_val` or `false_val` depending on the value of +/// `condition`, with a hint to the compiler that `condition` is unlikely to be +/// correctly predicted by a CPU’s branch predictor. +/// +/// This method is functionally equivalent to +/// ```ignore (this is just for illustrative purposes) +/// fn select_unpredictable<T>(b: bool, true_val: T, false_val: T) -> T { +/// if b { true_val } else { false_val } +/// } +/// ``` +/// but might generate different assembly. In particular, on platforms with +/// a conditional move or select instruction (like `cmov` on x86 or `csel` +/// on ARM) the optimizer might use these instructions to avoid branches, +/// which can benefit performance if the branch predictor is struggling +/// with predicting `condition`, such as in an implementation of binary +/// search. +/// +/// Note however that this lowering is not guaranteed (on any platform) and +/// should not be relied upon when trying to write cryptographic constant-time +/// code. Also be aware that this lowering might *decrease* performance if +/// `condition` is well-predictable. It is advisable to perform benchmarks to +/// tell if this function is useful. +/// +/// # Examples +/// +/// Distribute values evenly between two buckets: +/// ``` +/// #![feature(select_unpredictable)] +/// +/// use std::hash::BuildHasher; +/// use std::hint; +/// +/// fn append<H: BuildHasher>(hasher: &H, v: i32, bucket_one: &mut Vec<i32>, bucket_two: &mut Vec<i32>) { +/// let hash = hasher.hash_one(&v); +/// let bucket = hint::select_unpredictable(hash % 2 == 0, bucket_one, bucket_two); +/// bucket.push(v); +/// } +/// # let hasher = std::collections::hash_map::RandomState::new(); +/// # let mut bucket_one = Vec::new(); +/// # let mut bucket_two = Vec::new(); +/// # append(&hasher, 42, &mut bucket_one, &mut bucket_two); +/// # assert_eq!(bucket_one.len() + bucket_two.len(), 1); +/// ``` +#[inline(always)] +#[unstable(feature = "select_unpredictable", issue = "133962")] +pub fn select_unpredictable<T>(condition: bool, true_val: T, false_val: T) -> T { + // FIXME(https://github.com/rust-lang/unsafe-code-guidelines/issues/245): + // Change this to use ManuallyDrop instead. + let mut true_val = MaybeUninit::new(true_val); + let mut false_val = MaybeUninit::new(false_val); + // SAFETY: The value that is not selected is dropped, and the selected one + // is returned. This is necessary because the intrinsic doesn't drop the + // value that is not selected. + unsafe { + crate::intrinsics::select_unpredictable(!condition, &mut true_val, &mut false_val) + .assume_init_drop(); + crate::intrinsics::select_unpredictable(condition, true_val, false_val).assume_init() + } +} diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index afd6192d7c4..a01efb2adeb 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -5,14 +5,11 @@ //! //! # Const intrinsics //! -//! Note: any changes to the constness of intrinsics should be discussed with the language team. -//! This includes changes in the stability of the constness. -//! -//! In order to make an intrinsic usable at compile-time, it needs to be declared in the "new" -//! style, i.e. as a `#[rustc_intrinsic]` function, not inside an `extern` block. Then copy the -//! implementation from <https://github.com/rust-lang/miri/blob/master/src/intrinsics> to +//! In order to make an intrinsic unstable usable at compile-time, copy the implementation from +//! <https://github.com/rust-lang/miri/blob/master/src/intrinsics> to //! <https://github.com/rust-lang/rust/blob/master/compiler/rustc_const_eval/src/interpret/intrinsics.rs> -//! and make the intrinsic declaration a `const fn`. +//! and make the intrinsic declaration below a `const fn`. This should be done in coordination with +//! wg-const-eval. //! //! If an intrinsic is supposed to be used from a `const fn` with a `rustc_const_stable` attribute, //! `#[rustc_intrinsic_const_stable_indirect]` needs to be added to the intrinsic. Such a change requires @@ -1329,7 +1326,9 @@ pub const fn unlikely(b: bool) -> bool { /// Therefore, implementations must not require the user to uphold /// any safety invariants. /// -/// The public form of this instrinsic is [`bool::select_unpredictable`]. +/// The public form of this instrinsic is [`core::hint::select_unpredictable`]. +/// However unlike the public form, the intrinsic will not drop the value that +/// is not selected. #[unstable(feature = "core_intrinsics", issue = "none")] #[rustc_intrinsic] #[rustc_nounwind] @@ -2307,20 +2306,8 @@ pub unsafe fn truncf128(x: f128) -> f128; /// [`f16::round_ties_even`](../../std/primitive.f16.html#method.round_ties_even) #[rustc_intrinsic] #[rustc_nounwind] -#[cfg(not(bootstrap))] pub fn round_ties_even_f16(x: f16) -> f16; -/// To be removed on next bootstrap bump. -#[cfg(bootstrap)] -pub fn round_ties_even_f16(x: f16) -> f16 { - #[rustc_intrinsic] - #[rustc_nounwind] - unsafe fn rintf16(x: f16) -> f16; - - // SAFETY: this intrinsic isn't actually unsafe - unsafe { rintf16(x) } -} - /// Returns the nearest integer to an `f32`. Rounds half-way cases to the number with an even /// least significant digit. /// @@ -2328,20 +2315,8 @@ pub fn round_ties_even_f16(x: f16) -> f16 { /// [`f32::round_ties_even`](../../std/primitive.f32.html#method.round_ties_even) #[rustc_intrinsic] #[rustc_nounwind] -#[cfg(not(bootstrap))] pub fn round_ties_even_f32(x: f32) -> f32; -/// To be removed on next bootstrap bump. -#[cfg(bootstrap)] -pub fn round_ties_even_f32(x: f32) -> f32 { - #[rustc_intrinsic] - #[rustc_nounwind] - unsafe fn rintf32(x: f32) -> f32; - - // SAFETY: this intrinsic isn't actually unsafe - unsafe { rintf32(x) } -} - /// Provided for compatibility with stdarch. DO NOT USE. #[inline(always)] pub unsafe fn rintf32(x: f32) -> f32 { @@ -2355,20 +2330,8 @@ pub unsafe fn rintf32(x: f32) -> f32 { /// [`f64::round_ties_even`](../../std/primitive.f64.html#method.round_ties_even) #[rustc_intrinsic] #[rustc_nounwind] -#[cfg(not(bootstrap))] pub fn round_ties_even_f64(x: f64) -> f64; -/// To be removed on next bootstrap bump. -#[cfg(bootstrap)] -pub fn round_ties_even_f64(x: f64) -> f64 { - #[rustc_intrinsic] - #[rustc_nounwind] - unsafe fn rintf64(x: f64) -> f64; - - // SAFETY: this intrinsic isn't actually unsafe - unsafe { rintf64(x) } -} - /// Provided for compatibility with stdarch. DO NOT USE. #[inline(always)] pub unsafe fn rintf64(x: f64) -> f64 { @@ -2382,20 +2345,8 @@ pub unsafe fn rintf64(x: f64) -> f64 { /// [`f128::round_ties_even`](../../std/primitive.f128.html#method.round_ties_even) #[rustc_intrinsic] #[rustc_nounwind] -#[cfg(not(bootstrap))] pub fn round_ties_even_f128(x: f128) -> f128; -/// To be removed on next bootstrap bump. -#[cfg(bootstrap)] -pub fn round_ties_even_f128(x: f128) -> f128 { - #[rustc_intrinsic] - #[rustc_nounwind] - unsafe fn rintf128(x: f128) -> f128; - - // SAFETY: this intrinsic isn't actually unsafe - unsafe { rintf128(x) } -} - /// Returns the nearest integer to an `f16`. Rounds half-way cases away from zero. /// /// The stabilized version of this intrinsic is @@ -2679,13 +2630,15 @@ pub const fn bswap<T: Copy>(x: T) -> T; #[rustc_intrinsic] pub const fn bitreverse<T: Copy>(x: T) -> T; -/// Does a three-way comparison between the two integer arguments. +/// Does a three-way comparison between the two arguments, +/// which must be of character or integer (signed or unsigned) type. /// -/// This is included as an intrinsic as it's useful to let it be one thing -/// in MIR, rather than the multiple checks and switches that make its IR -/// large and difficult to optimize. +/// This was originally added because it greatly simplified the MIR in `cmp` +/// implementations, and then LLVM 20 added a backend intrinsic for it too. /// /// The stabilized version of this intrinsic is [`Ord::cmp`]. +#[rustc_intrinsic_const_stable_indirect] +#[rustc_nounwind] #[rustc_intrinsic] pub const fn three_way_compare<T: Copy>(lhs: T, rhss: T) -> crate::cmp::Ordering; @@ -2786,6 +2739,7 @@ pub const fn carrying_mul_add<T: ~const fallback::CarryingMulAdd<Unsigned = U>, /// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1` /// /// This intrinsic does not have a stable counterpart. +#[rustc_intrinsic_const_stable_indirect] #[rustc_nounwind] #[rustc_intrinsic] pub const unsafe fn exact_div<T: Copy>(x: T, y: T) -> T; @@ -3450,20 +3404,62 @@ pub const fn contract_checks() -> bool { /// /// By default, if `contract_checks` is enabled, this will panic with no unwind if the condition /// returns false. -#[unstable(feature = "contracts_internals", issue = "128044" /* compiler-team#759 */)] +/// +/// Note that this function is a no-op during constant evaluation. +#[unstable(feature = "contracts_internals", issue = "128044")] +// Calls to this function get inserted by an AST expansion pass, which uses the equivalent of +// `#[allow_internal_unstable]` to allow using `contracts_internals` functions. Const-checking +// doesn't honor `#[allow_internal_unstable]`, so for the const feature gate we use the user-facing +// `contracts` feature rather than the perma-unstable `contracts_internals` +#[rustc_const_unstable(feature = "contracts", issue = "128044")] #[lang = "contract_check_requires"] #[rustc_intrinsic] -pub fn contract_check_requires<C: Fn() -> bool>(cond: C) { - if contract_checks() && !cond() { - // Emit no unwind panic in case this was a safety requirement. - crate::panicking::panic_nounwind("failed requires check"); - } +pub const fn contract_check_requires<C: Fn() -> bool + Copy>(cond: C) { + const_eval_select!( + @capture[C: Fn() -> bool + Copy] { cond: C } : + if const { + // Do nothing + } else { + if contract_checks() && !cond() { + // Emit no unwind panic in case this was a safety requirement. + crate::panicking::panic_nounwind("failed requires check"); + } + } + ) } /// Check if the post-condition `cond` has been met. /// /// By default, if `contract_checks` is enabled, this will panic with no unwind if the condition /// returns false. +/// +/// Note that this function is a no-op during constant evaluation. +#[cfg(not(bootstrap))] +#[unstable(feature = "contracts_internals", issue = "128044")] +// Similar to `contract_check_requires`, we need to use the user-facing +// `contracts` feature rather than the perma-unstable `contracts_internals`. +// Const-checking doesn't honor allow_internal_unstable logic used by contract expansion. +#[rustc_const_unstable(feature = "contracts", issue = "128044")] +#[lang = "contract_check_ensures"] +#[rustc_intrinsic] +pub const fn contract_check_ensures<C: Fn(&Ret) -> bool + Copy, Ret>(cond: C, ret: Ret) -> Ret { + const_eval_select!( + @capture[C: Fn(&Ret) -> bool + Copy, Ret] { cond: C, ret: Ret } -> Ret : + if const { + // Do nothing + ret + } else { + if contract_checks() && !cond(&ret) { + // Emit no unwind panic in case this was a safety requirement. + crate::panicking::panic_nounwind("failed ensures check"); + } + ret + } + ) +} + +/// This is the old version of contract_check_ensures kept here for bootstrap only. +#[cfg(bootstrap)] #[unstable(feature = "contracts_internals", issue = "128044" /* compiler-team#759 */)] #[rustc_intrinsic] pub fn contract_check_ensures<'a, Ret, C: Fn(&'a Ret) -> bool>(ret: &'a Ret, cond: C) { @@ -3723,7 +3719,7 @@ pub const fn ptr_metadata<P: ptr::Pointee<Metadata = M> + ?Sized, M>(ptr: *const /// [`Vec::append`]: ../../std/vec/struct.Vec.html#method.append #[doc(alias = "memcpy")] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_allowed_through_unstable_modules = "import this function via `std::mem` instead"] +#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"] #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.83.0")] #[inline(always)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces @@ -3826,7 +3822,7 @@ pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: us /// ``` #[doc(alias = "memmove")] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_allowed_through_unstable_modules = "import this function via `std::mem` instead"] +#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"] #[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.83.0")] #[inline(always)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces @@ -3906,7 +3902,7 @@ pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) { /// ``` #[doc(alias = "memset")] #[stable(feature = "rust1", since = "1.0.0")] -#[rustc_allowed_through_unstable_modules = "import this function via `std::mem` instead"] +#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"] #[rustc_const_stable(feature = "const_ptr_write", since = "1.83.0")] #[inline(always)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs index ae6e1a779ed..9ac6ee85535 100644 --- a/library/core/src/intrinsics/simd.rs +++ b/library/core/src/intrinsics/simd.rs @@ -4,7 +4,7 @@ /// Inserts an element into a vector, returning the updated vector. /// -/// `T` must be a vector with element type `U`. +/// `T` must be a vector with element type `U`, and `idx` must be `const`. /// /// # Safety /// @@ -15,15 +15,48 @@ pub const unsafe fn simd_insert<T, U>(x: T, idx: u32, val: U) -> T; /// Extracts an element from a vector. /// -/// `T` must be a vector with element type `U`. +/// `T` must be a vector with element type `U`, and `idx` must be `const`. /// /// # Safety /// -/// `idx` must be in-bounds of the vector. +/// `idx` must be const and in-bounds of the vector. #[rustc_intrinsic] #[rustc_nounwind] pub const unsafe fn simd_extract<T, U>(x: T, idx: u32) -> U; +/// Inserts an element into a vector, returning the updated vector. +/// +/// `T` must be a vector with element type `U`. +/// +/// If the index is `const`, [`simd_insert`] may emit better assembly. +/// +/// # Safety +/// +/// `idx` must be in-bounds of the vector. +#[rustc_nounwind] +#[cfg_attr(not(bootstrap), rustc_intrinsic)] +pub unsafe fn simd_insert_dyn<T, U>(mut x: T, idx: u32, val: U) -> T { + // SAFETY: `idx` must be in-bounds + unsafe { (&raw mut x).cast::<U>().add(idx as usize).write(val) } + x +} + +/// Extracts an element from a vector. +/// +/// `T` must be a vector with element type `U`. +/// +/// If the index is `const`, [`simd_extract`] may emit better assembly. +/// +/// # Safety +/// +/// `idx` must be in-bounds of the vector. +#[rustc_nounwind] +#[cfg_attr(not(bootstrap), rustc_intrinsic)] +pub unsafe fn simd_extract_dyn<T, U>(x: T, idx: u32) -> U { + // SAFETY: `idx` must be in-bounds + unsafe { (&raw const x).cast::<U>().add(idx as usize).read() } +} + /// Adds two simd vectors elementwise. /// /// `T` must be a vector of integers or floats. diff --git a/library/core/src/iter/adapters/cloned.rs b/library/core/src/iter/adapters/cloned.rs index aea6d64281a..72d74628971 100644 --- a/library/core/src/iter/adapters/cloned.rs +++ b/library/core/src/iter/adapters/cloned.rs @@ -1,5 +1,6 @@ use core::num::NonZero; +use crate::cmp::Ordering; use crate::iter::adapters::zip::try_get_unchecked; use crate::iter::adapters::{SourceIter, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; use crate::iter::{FusedIterator, InPlaceIterable, TrustedLen, UncheckedIterator}; @@ -41,13 +42,31 @@ where self.it.next().cloned() } + #[inline] fn size_hint(&self) -> (usize, Option<usize>) { self.it.size_hint() } + #[inline] + fn count(self) -> usize { + self.it.count() + } + + fn last(self) -> Option<T> { + self.it.last().cloned() + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + self.it.advance_by(n) + } + + fn nth(&mut self, n: usize) -> Option<T> { + self.it.nth(n).cloned() + } + fn try_fold<B, F, R>(&mut self, init: B, f: F) -> R where - Self: Sized, F: FnMut(B, Self::Item) -> R, R: Try<Output = B>, { @@ -61,6 +80,58 @@ where self.it.map(T::clone).fold(init, f) } + fn find<P>(&mut self, mut predicate: P) -> Option<Self::Item> + where + P: FnMut(&Self::Item) -> bool, + { + self.it.find(move |x| predicate(&x)).cloned() + } + + fn max_by<F>(self, mut compare: F) -> Option<Self::Item> + where + F: FnMut(&Self::Item, &Self::Item) -> Ordering, + { + self.it.max_by(move |&x, &y| compare(x, y)).cloned() + } + + fn min_by<F>(self, mut compare: F) -> Option<Self::Item> + where + F: FnMut(&Self::Item, &Self::Item) -> Ordering, + { + self.it.min_by(move |&x, &y| compare(x, y)).cloned() + } + + fn cmp<O>(self, other: O) -> Ordering + where + O: IntoIterator<Item = Self::Item>, + Self::Item: Ord, + { + self.it.cmp_by(other, |x, y| x.cmp(&y)) + } + + fn partial_cmp<O>(self, other: O) -> Option<Ordering> + where + O: IntoIterator, + Self::Item: PartialOrd<O::Item>, + { + self.it.partial_cmp_by(other, |x, y| x.partial_cmp(&y)) + } + + fn eq<O>(self, other: O) -> bool + where + O: IntoIterator, + Self::Item: PartialEq<O::Item>, + { + self.it.eq_by(other, |x, y| x == &y) + } + + fn is_sorted_by<F>(self, mut compare: F) -> bool + where + F: FnMut(&Self::Item, &Self::Item) -> bool, + { + self.it.is_sorted_by(move |&x, &y| compare(x, y)) + } + unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> T where Self: TrustedRandomAccessNoCoerce, @@ -81,9 +152,13 @@ where self.it.next_back().cloned() } + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + self.it.advance_back_by(n) + } + fn try_rfold<B, F, R>(&mut self, init: B, f: F) -> R where - Self: Sized, F: FnMut(B, Self::Item) -> R, R: Try<Output = B>, { @@ -96,6 +171,13 @@ where { self.it.map(T::clone).rfold(init, f) } + + fn rfind<P>(&mut self, mut predicate: P) -> Option<Self::Item> + where + P: FnMut(&Self::Item) -> bool, + { + self.it.rfind(move |x| predicate(&x)).cloned() + } } #[stable(feature = "iter_cloned", since = "1.1.0")] @@ -104,10 +186,12 @@ where I: ExactSizeIterator<Item = &'a T>, T: Clone, { + #[inline] fn len(&self) -> usize { self.it.len() } + #[inline] fn is_empty(&self) -> bool { self.it.is_empty() } diff --git a/library/core/src/iter/adapters/copied.rs b/library/core/src/iter/adapters/copied.rs index 23e4e25ab53..73913aa34a9 100644 --- a/library/core/src/iter/adapters/copied.rs +++ b/library/core/src/iter/adapters/copied.rs @@ -1,3 +1,4 @@ +use crate::cmp::Ordering; use crate::iter::adapters::zip::try_get_unchecked; use crate::iter::adapters::{SourceIter, TrustedRandomAccess, TrustedRandomAccessNoCoerce}; use crate::iter::{FusedIterator, InPlaceIterable, TrustedLen}; @@ -48,20 +49,35 @@ where fn next_chunk<const N: usize>( &mut self, - ) -> Result<[Self::Item; N], array::IntoIter<Self::Item, N>> - where - Self: Sized, - { + ) -> Result<[Self::Item; N], array::IntoIter<Self::Item, N>> { <I as SpecNextChunk<'_, N, T>>::spec_next_chunk(&mut self.it) } + #[inline] fn size_hint(&self) -> (usize, Option<usize>) { self.it.size_hint() } + #[inline] + fn count(self) -> usize { + self.it.count() + } + + fn last(self) -> Option<T> { + self.it.last().copied() + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + self.it.advance_by(n) + } + + fn nth(&mut self, n: usize) -> Option<T> { + self.it.nth(n).copied() + } + fn try_fold<B, F, R>(&mut self, init: B, f: F) -> R where - Self: Sized, F: FnMut(B, Self::Item) -> R, R: Try<Output = B>, { @@ -75,21 +91,56 @@ where self.it.fold(init, copy_fold(f)) } - fn nth(&mut self, n: usize) -> Option<T> { - self.it.nth(n).copied() + fn find<P>(&mut self, mut predicate: P) -> Option<Self::Item> + where + P: FnMut(&Self::Item) -> bool, + { + self.it.find(move |x| predicate(&x)).copied() } - fn last(self) -> Option<T> { - self.it.last().copied() + fn max_by<F>(self, mut compare: F) -> Option<Self::Item> + where + F: FnMut(&Self::Item, &Self::Item) -> Ordering, + { + self.it.max_by(move |&x, &y| compare(x, y)).copied() } - fn count(self) -> usize { - self.it.count() + fn min_by<F>(self, mut compare: F) -> Option<Self::Item> + where + F: FnMut(&Self::Item, &Self::Item) -> Ordering, + { + self.it.min_by(move |&x, &y| compare(x, y)).copied() } - #[inline] - fn advance_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { - self.it.advance_by(n) + fn cmp<O>(self, other: O) -> Ordering + where + O: IntoIterator<Item = Self::Item>, + Self::Item: Ord, + { + self.it.cmp_by(other, |x, y| x.cmp(&y)) + } + + fn partial_cmp<O>(self, other: O) -> Option<Ordering> + where + O: IntoIterator, + Self::Item: PartialOrd<O::Item>, + { + self.it.partial_cmp_by(other, |x, y| x.partial_cmp(&y)) + } + + fn eq<O>(self, other: O) -> bool + where + O: IntoIterator, + Self::Item: PartialEq<O::Item>, + { + self.it.eq_by(other, |x, y| x == &y) + } + + fn is_sorted_by<F>(self, mut compare: F) -> bool + where + F: FnMut(&Self::Item, &Self::Item) -> bool, + { + self.it.is_sorted_by(move |&x, &y| compare(x, y)) } unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> T @@ -112,9 +163,13 @@ where self.it.next_back().copied() } + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { + self.it.advance_back_by(n) + } + fn try_rfold<B, F, R>(&mut self, init: B, f: F) -> R where - Self: Sized, F: FnMut(B, Self::Item) -> R, R: Try<Output = B>, { @@ -128,9 +183,11 @@ where self.it.rfold(init, copy_fold(f)) } - #[inline] - fn advance_back_by(&mut self, n: usize) -> Result<(), NonZero<usize>> { - self.it.advance_back_by(n) + fn rfind<P>(&mut self, mut predicate: P) -> Option<Self::Item> + where + P: FnMut(&Self::Item) -> bool, + { + self.it.rfind(move |x| predicate(&x)).copied() } } @@ -140,10 +197,12 @@ where I: ExactSizeIterator<Item = &'a T>, T: Copy, { + #[inline] fn len(&self) -> usize { self.it.len() } + #[inline] fn is_empty(&self) -> bool { self.it.is_empty() } diff --git a/library/core/src/iter/adapters/enumerate.rs b/library/core/src/iter/adapters/enumerate.rs index f9c388e8564..bd093e279c3 100644 --- a/library/core/src/iter/adapters/enumerate.rs +++ b/library/core/src/iter/adapters/enumerate.rs @@ -36,7 +36,7 @@ where /// /// The method does no guarding against overflows, so enumerating more than /// `usize::MAX` elements either produces the wrong result or panics. If - /// debug assertions are enabled, a panic is guaranteed. + /// overflow checks are enabled, a panic is guaranteed. /// /// # Panics /// diff --git a/library/core/src/iter/traits/iterator.rs b/library/core/src/iter/traits/iterator.rs index 3bbb52fdbcb..d9534a44598 100644 --- a/library/core/src/iter/traits/iterator.rs +++ b/library/core/src/iter/traits/iterator.rs @@ -199,7 +199,7 @@ pub trait Iterator { /// /// The method does no guarding against overflows, so counting elements of /// an iterator with more than [`usize::MAX`] elements either produces the - /// wrong result or panics. If debug assertions are enabled, a panic is + /// wrong result or panics. If overflow checks are enabled, a panic is /// guaranteed. /// /// # Panics @@ -931,7 +931,7 @@ pub trait Iterator { /// /// The method does no guarding against overflows, so enumerating more than /// [`usize::MAX`] elements either produces the wrong result or panics. If - /// debug assertions are enabled, a panic is guaranteed. + /// overflow checks are enabled, a panic is guaranteed. /// /// # Panics /// @@ -2964,7 +2964,7 @@ pub trait Iterator { /// /// The method does no guarding against overflows, so if there are more /// than [`usize::MAX`] non-matching elements, it either produces the wrong - /// result or panics. If debug assertions are enabled, a panic is + /// result or panics. If overflow checks are enabled, a panic is /// guaranteed. /// /// # Panics @@ -3516,7 +3516,7 @@ pub trait Iterator { /// # Panics /// /// When calling `sum()` and a primitive integer type is being returned, this - /// method will panic if the computation overflows and debug assertions are + /// method will panic if the computation overflows and overflow checks are /// enabled. /// /// # Examples @@ -3550,7 +3550,7 @@ pub trait Iterator { /// # Panics /// /// When calling `product()` and a primitive integer type is being returned, - /// method will panic if the computation overflows and debug assertions are + /// method will panic if the computation overflows and overflow checks are /// enabled. /// /// # Examples diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index dc06aa4c38d..9625475e617 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -101,7 +101,6 @@ #![feature(bstr)] #![feature(bstr_internals)] #![feature(cfg_match)] -#![feature(closure_track_caller)] #![feature(const_carrying_mul_add)] #![feature(const_eval_select)] #![feature(core_intrinsics)] @@ -127,6 +126,7 @@ #![feature(ub_checks)] #![feature(unchecked_neg)] #![feature(unchecked_shifts)] +#![feature(unsafe_pinned)] #![feature(utf16_extra)] #![feature(variant_count)] // tidy-alphabetical-end @@ -169,7 +169,6 @@ #![feature(negative_impls)] #![feature(never_type)] #![feature(no_core)] -#![feature(no_sanitize)] #![feature(optimize_attribute)] #![feature(prelude_import)] #![feature(repr_simd)] diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs index 5f200b31d1a..330b4098764 100644 --- a/library/core/src/macros/mod.rs +++ b/library/core/src/macros/mod.rs @@ -237,9 +237,10 @@ pub macro assert_matches { /// ``` #[unstable(feature = "cfg_match", issue = "115585")] #[rustc_diagnostic_item = "cfg_match"] +#[rustc_macro_transparency = "semitransparent"] pub macro cfg_match { ({ $($tt:tt)* }) => {{ - cfg_match! { $($tt)* } + $crate::cfg_match! { $($tt)* } }}, (_ => { $($output:tt)* }) => { $($output)* @@ -249,10 +250,10 @@ pub macro cfg_match { $($( $rest:tt )+)? ) => { #[cfg($cfg)] - cfg_match! { _ => $output } + $crate::cfg_match! { _ => $output } $( #[cfg(not($cfg))] - cfg_match! { $($rest)+ } + $crate::cfg_match! { $($rest)+ } )? }, } @@ -1753,7 +1754,6 @@ pub(crate) mod builtin { reason = "`type_alias_impl_trait` has open design concerns" )] #[rustc_builtin_macro] - #[cfg(not(bootstrap))] pub macro define_opaque($($tt:tt)*) { /* compiler built-in */ } diff --git a/library/core/src/marker.rs b/library/core/src/marker.rs index 68011310d2c..9dc20beda6c 100644 --- a/library/core/src/marker.rs +++ b/library/core/src/marker.rs @@ -17,6 +17,7 @@ use crate::cell::UnsafeCell; use crate::cmp; use crate::fmt::Debug; use crate::hash::{Hash, Hasher}; +use crate::pin::UnsafePinned; /// Implements a given marker trait for multiple types at the same time. /// @@ -878,6 +879,23 @@ marker_impls! { {T: ?Sized} &mut T, } +/// Used to determine whether a type contains any `UnsafePinned` (or `PhantomPinned`) internally, +/// but not through an indirection. This affects, for example, whether we emit `noalias` metadata +/// for `&mut T` or not. +/// +/// This is part of [RFC 3467](https://rust-lang.github.io/rfcs/3467-unsafe-pinned.html), and is +/// tracked by [#125735](https://github.com/rust-lang/rust/issues/125735). +#[cfg_attr(not(bootstrap), lang = "unsafe_unpin")] +#[cfg_attr(bootstrap, allow(dead_code))] +pub(crate) unsafe auto trait UnsafeUnpin {} + +impl<T: ?Sized> !UnsafeUnpin for UnsafePinned<T> {} +unsafe impl<T: ?Sized> UnsafeUnpin for PhantomData<T> {} +unsafe impl<T: ?Sized> UnsafeUnpin for *const T {} +unsafe impl<T: ?Sized> UnsafeUnpin for *mut T {} +unsafe impl<T: ?Sized> UnsafeUnpin for &T {} +unsafe impl<T: ?Sized> UnsafeUnpin for &mut T {} + /// Types that do not require any pinning guarantees. /// /// For information on what "pinning" is, see the [`pin` module] documentation. @@ -953,6 +971,11 @@ pub auto trait Unpin {} /// A marker type which does not implement `Unpin`. /// /// If a type contains a `PhantomPinned`, it will not implement `Unpin` by default. +// +// FIXME(unsafe_pinned): This is *not* a stable guarantee we want to make, at least not yet. +// Note that for backwards compatibility with the new [`UnsafePinned`] wrapper type, placing this +// marker in your struct acts as if you wrapped the entire struct in an `UnsafePinned`. This type +// will likely eventually be deprecated, and all new code should be using `UnsafePinned` instead. #[stable(feature = "pin", since = "1.33.0")] #[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct PhantomPinned; @@ -960,6 +983,12 @@ pub struct PhantomPinned; #[stable(feature = "pin", since = "1.33.0")] impl !Unpin for PhantomPinned {} +// This is a small hack to allow existing code which uses PhantomPinned to opt-out of noalias to +// continue working. Ideally PhantomPinned could just wrap an `UnsafePinned<()>` to get the same +// effect, but we can't add a new field to an already stable unit struct -- that would be a breaking +// change. +impl !UnsafeUnpin for PhantomPinned {} + marker_impls! { #[stable(feature = "pin", since = "1.33.0")] Unpin for diff --git a/library/core/src/net/socket_addr.rs b/library/core/src/net/socket_addr.rs index 21753d00924..936f9f64930 100644 --- a/library/core/src/net/socket_addr.rs +++ b/library/core/src/net/socket_addr.rs @@ -210,7 +210,7 @@ impl SocketAddr { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_ip(&mut self, new_ip: IpAddr) { // `match (*self, new_ip)` would have us mutate a copy of self only to throw it away. match (self, new_ip) { @@ -254,7 +254,7 @@ impl SocketAddr { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_port(&mut self, new_port: u16) { match *self { SocketAddr::V4(ref mut a) => a.set_port(new_port), @@ -360,7 +360,7 @@ impl SocketAddrV4 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_ip(&mut self, new_ip: Ipv4Addr) { self.ip = new_ip; } @@ -396,7 +396,7 @@ impl SocketAddrV4 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_port(&mut self, new_port: u16) { self.port = new_port; } @@ -458,7 +458,7 @@ impl SocketAddrV6 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_ip(&mut self, new_ip: Ipv6Addr) { self.ip = new_ip; } @@ -494,7 +494,7 @@ impl SocketAddrV6 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_port(&mut self, new_port: u16) { self.port = new_port; } @@ -542,7 +542,7 @@ impl SocketAddrV6 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_flowinfo(&mut self, new_flowinfo: u32) { self.flowinfo = new_flowinfo; } @@ -585,7 +585,7 @@ impl SocketAddrV6 { /// ``` #[inline] #[stable(feature = "sockaddr_setters", since = "1.9.0")] - #[rustc_const_stable(feature = "const_sockaddr_setters", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_sockaddr_setters", since = "1.87.0")] pub const fn set_scope_id(&mut self, new_scope_id: u32) { self.scope_id = new_scope_id; } diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index 08c34e852da..d3d1eebc227 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -224,14 +224,16 @@ impl f128 { /// Not a Number (NaN). /// - /// Note that IEEE 754 doesn't define just a single NaN value; - /// a plethora of bit patterns are considered to be NaN. - /// Furthermore, the standard makes a difference - /// between a "signaling" and a "quiet" NaN, - /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). - /// This constant isn't guaranteed to equal to any specific NaN bitpattern, - /// and the stability of its representation over Rust versions - /// and target platforms isn't guaranteed. + /// Note that IEEE 754 doesn't define just a single NaN value; a plethora of bit patterns are + /// considered to be NaN. Furthermore, the standard makes a difference between a "signaling" and + /// a "quiet" NaN, and allows inspecting its "payload" (the unspecified bits in the bit pattern) + /// and its sign. See the [specification of NaN bit patterns](f32#nan-bit-patterns) for more + /// info. + /// + /// This constant is guaranteed to be a quiet NaN (on targets that follow the Rust assumptions + /// that the quiet/signaling bit being set to 1 indicates a quiet NaN). Beyond that, nothing is + /// guaranteed about the specific bit pattern chosen here: both payload and sign are arbitrary. + /// The concrete bit pattern may change across Rust versions and target platforms. #[allow(clippy::eq_op)] #[rustc_diagnostic_item = "f128_nan"] #[unstable(feature = "f128", issue = "116909")] diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index a33e5f53014..dceb30177e6 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -219,14 +219,16 @@ impl f16 { /// Not a Number (NaN). /// - /// Note that IEEE 754 doesn't define just a single NaN value; - /// a plethora of bit patterns are considered to be NaN. - /// Furthermore, the standard makes a difference - /// between a "signaling" and a "quiet" NaN, - /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). - /// This constant isn't guaranteed to equal to any specific NaN bitpattern, - /// and the stability of its representation over Rust versions - /// and target platforms isn't guaranteed. + /// Note that IEEE 754 doesn't define just a single NaN value; a plethora of bit patterns are + /// considered to be NaN. Furthermore, the standard makes a difference between a "signaling" and + /// a "quiet" NaN, and allows inspecting its "payload" (the unspecified bits in the bit pattern) + /// and its sign. See the [specification of NaN bit patterns](f32#nan-bit-patterns) for more + /// info. + /// + /// This constant is guaranteed to be a quiet NaN (on targets that follow the Rust assumptions + /// that the quiet/signaling bit being set to 1 indicates a quiet NaN). Beyond that, nothing is + /// guaranteed about the specific bit pattern chosen here: both payload and sign are arbitrary. + /// The concrete bit pattern may change across Rust versions and target platforms. #[allow(clippy::eq_op)] #[rustc_diagnostic_item = "f16_nan"] #[unstable(feature = "f16", issue = "116909")] diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index e473fac0393..c97dbfb63ae 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -470,14 +470,16 @@ impl f32 { /// Not a Number (NaN). /// - /// Note that IEEE 754 doesn't define just a single NaN value; - /// a plethora of bit patterns are considered to be NaN. - /// Furthermore, the standard makes a difference - /// between a "signaling" and a "quiet" NaN, - /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). - /// This constant isn't guaranteed to equal to any specific NaN bitpattern, - /// and the stability of its representation over Rust versions - /// and target platforms isn't guaranteed. + /// Note that IEEE 754 doesn't define just a single NaN value; a plethora of bit patterns are + /// considered to be NaN. Furthermore, the standard makes a difference between a "signaling" and + /// a "quiet" NaN, and allows inspecting its "payload" (the unspecified bits in the bit pattern) + /// and its sign. See the [specification of NaN bit patterns](f32#nan-bit-patterns) for more + /// info. + /// + /// This constant is guaranteed to be a quiet NaN (on targets that follow the Rust assumptions + /// that the quiet/signaling bit being set to 1 indicates a quiet NaN). Beyond that, nothing is + /// guaranteed about the specific bit pattern chosen here: both payload and sign are arbitrary. + /// The concrete bit pattern may change across Rust versions and target platforms. #[stable(feature = "assoc_int_consts", since = "1.43.0")] #[rustc_diagnostic_item = "f32_nan"] #[allow(clippy::eq_op)] diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index 6522a80b0b7..91affdb3794 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -469,14 +469,16 @@ impl f64 { /// Not a Number (NaN). /// - /// Note that IEEE 754 doesn't define just a single NaN value; - /// a plethora of bit patterns are considered to be NaN. - /// Furthermore, the standard makes a difference - /// between a "signaling" and a "quiet" NaN, - /// and allows inspecting its "payload" (the unspecified bits in the bit pattern). - /// This constant isn't guaranteed to equal to any specific NaN bitpattern, - /// and the stability of its representation over Rust versions - /// and target platforms isn't guaranteed. + /// Note that IEEE 754 doesn't define just a single NaN value; a plethora of bit patterns are + /// considered to be NaN. Furthermore, the standard makes a difference between a "signaling" and + /// a "quiet" NaN, and allows inspecting its "payload" (the unspecified bits in the bit pattern) + /// and its sign. See the [specification of NaN bit patterns](f32#nan-bit-patterns) for more + /// info. + /// + /// This constant is guaranteed to be a quiet NaN (on targets that follow the Rust assumptions + /// that the quiet/signaling bit being set to 1 indicates a quiet NaN). Beyond that, nothing is + /// guaranteed about the specific bit pattern chosen here: both payload and sign are arbitrary. + /// The concrete bit pattern may change across Rust versions and target platforms. #[rustc_diagnostic_item = "f64_nan"] #[stable(feature = "assoc_int_consts", since = "1.43.0")] #[allow(clippy::eq_op)] diff --git a/library/core/src/num/int_macros.rs b/library/core/src/num/int_macros.rs index a72ca4bcb05..05d8216ac27 100644 --- a/library/core/src/num/int_macros.rs +++ b/library/core/src/num/int_macros.rs @@ -244,8 +244,8 @@ macro_rules! int_impl { /// #[doc = concat!("assert_eq!(n.cast_unsigned(), ", stringify!($UnsignedT), "::MAX);")] /// ``` - #[stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "integer_sign_cast", since = "1.87.0")] + #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline(always)] @@ -1355,8 +1355,8 @@ macro_rules! int_impl { #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".unbounded_shl(4), 0x10);")] #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".unbounded_shl(129), 0);")] /// ``` - #[stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "unbounded_shifts", since = "1.87.0")] + #[rustc_const_stable(feature = "unbounded_shifts", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] @@ -1478,8 +1478,8 @@ macro_rules! int_impl { #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".unbounded_shr(129), 0);")] #[doc = concat!("assert_eq!(", stringify!($SelfT), "::MIN.unbounded_shr(129), -1);")] /// ``` - #[stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "unbounded_shifts", since = "1.87.0")] + #[rustc_const_stable(feature = "unbounded_shifts", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] @@ -3571,10 +3571,7 @@ macro_rules! int_impl { // so delegate it to `Ord` which is already producing -1/0/+1 // exactly like we need and can be the place to deal with the complexity. - // FIXME(const-hack): replace with cmp - if self < 0 { -1 } - else if self == 0 { 0 } - else { 1 } + crate::intrinsics::three_way_compare(self, 0) as Self } /// Returns `true` if `self` is positive and `false` if the number is zero or diff --git a/library/core/src/num/mod.rs b/library/core/src/num/mod.rs index 151e128cd78..348457e2c00 100644 --- a/library/core/src/num/mod.rs +++ b/library/core/src/num/mod.rs @@ -99,8 +99,8 @@ macro_rules! i8_xe_bytes_doc { **Note**: This function is meaningless on `i8`. Byte order does not exist as a concept for byte-sized integers. This function is only provided in symmetry -with larger integer types. You can cast from and to `u8` using `as i8` and `as -u8`. +with larger integer types. You can cast from and to `u8` using +[`cast_signed`](u8::cast_signed) and [`cast_unsigned`](Self::cast_unsigned). " }; @@ -169,8 +169,8 @@ macro_rules! midpoint_impl { #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".midpoint(-7), -3);")] #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".midpoint(7), 3);")] /// ``` - #[stable(feature = "num_midpoint_signed", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "num_midpoint_signed", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "num_midpoint_signed", since = "1.87.0")] + #[rustc_const_stable(feature = "num_midpoint_signed", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] @@ -221,8 +221,8 @@ macro_rules! midpoint_impl { #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".midpoint(-7), -3);")] #[doc = concat!("assert_eq!(0", stringify!($SelfT), ".midpoint(7), 3);")] /// ``` - #[stable(feature = "num_midpoint_signed", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "num_midpoint_signed", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "num_midpoint_signed", since = "1.87.0")] + #[rustc_const_stable(feature = "num_midpoint_signed", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] diff --git a/library/core/src/num/nonzero.rs b/library/core/src/num/nonzero.rs index 7585ec140e3..1b79112aec1 100644 --- a/library/core/src/num/nonzero.rs +++ b/library/core/src/num/nonzero.rs @@ -1704,8 +1704,8 @@ macro_rules! nonzero_integer_signedness_dependent_methods { /// #[doc = concat!("assert_eq!(n.cast_signed(), NonZero::new(-1", stringify!($Sint), ").unwrap());")] /// ``` - #[stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "integer_sign_cast", since = "1.87.0")] + #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline(always)] @@ -2143,8 +2143,8 @@ macro_rules! nonzero_integer_signedness_dependent_methods { /// #[doc = concat!("assert_eq!(n.cast_unsigned(), NonZero::<", stringify!($Uint), ">::MAX);")] /// ``` - #[stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "integer_sign_cast", since = "1.87.0")] + #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline(always)] diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index 58689275839..3678bb091e7 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -273,8 +273,8 @@ macro_rules! uint_impl { /// #[doc = concat!("assert_eq!(n.cast_signed(), -1", stringify!($SignedT), ");")] /// ``` - #[stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "integer_sign_cast", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "integer_sign_cast", since = "1.87.0")] + #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline(always)] @@ -1616,8 +1616,8 @@ macro_rules! uint_impl { #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".unbounded_shl(4), 0x10);")] #[doc = concat!("assert_eq!(0x1", stringify!($SelfT), ".unbounded_shl(129), 0);")] /// ``` - #[stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "unbounded_shifts", since = "1.87.0")] + #[rustc_const_stable(feature = "unbounded_shifts", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] @@ -1737,8 +1737,8 @@ macro_rules! uint_impl { #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".unbounded_shr(4), 0x1);")] #[doc = concat!("assert_eq!(0x10", stringify!($SelfT), ".unbounded_shr(129), 0);")] /// ``` - #[stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "unbounded_shifts", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "unbounded_shifts", since = "1.87.0")] + #[rustc_const_stable(feature = "unbounded_shifts", since = "1.87.0")] #[must_use = "this returns the result of the operation, \ without modifying the original"] #[inline] @@ -3331,8 +3331,8 @@ macro_rules! uint_impl { #[doc = concat!("assert!(0_", stringify!($SelfT), ".is_multiple_of(0));")] #[doc = concat!("assert!(!6_", stringify!($SelfT), ".is_multiple_of(0));")] /// ``` - #[stable(feature = "unsigned_is_multiple_of", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "unsigned_is_multiple_of", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "unsigned_is_multiple_of", since = "1.87.0")] + #[rustc_const_stable(feature = "unsigned_is_multiple_of", since = "1.87.0")] #[must_use] #[inline] #[rustc_inherit_overflow_checks] @@ -3343,7 +3343,7 @@ macro_rules! uint_impl { } } - /// Returns `true` if and only if `self == 2^k` for some `k`. + /// Returns `true` if and only if `self == 2^k` for some unsigned integer `k`. /// /// # Examples /// diff --git a/library/core/src/ops/index_range.rs b/library/core/src/ops/index_range.rs index b82184b15b2..c645c996eb7 100644 --- a/library/core/src/ops/index_range.rs +++ b/library/core/src/ops/index_range.rs @@ -1,5 +1,6 @@ use crate::iter::{FusedIterator, TrustedLen}; use crate::num::NonZero; +use crate::ops::{NeverShortCircuit, Try}; use crate::ub_checks; /// Like a `Range<usize>`, but with a safety invariant that `start <= end`. @@ -112,6 +113,12 @@ impl IndexRange { self.end = mid; suffix } + + #[inline] + fn assume_range(&self) { + // SAFETY: This is the type invariant + unsafe { crate::hint::assert_unchecked(self.start <= self.end) } + } } impl Iterator for IndexRange { @@ -138,6 +145,30 @@ impl Iterator for IndexRange { let taken = self.take_prefix(n); NonZero::new(n - taken.len()).map_or(Ok(()), Err) } + + #[inline] + fn fold<B, F: FnMut(B, usize) -> B>(mut self, init: B, f: F) -> B { + self.try_fold(init, NeverShortCircuit::wrap_mut_2(f)).0 + } + + #[inline] + fn try_fold<B, F, R>(&mut self, mut accum: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try<Output = B>, + { + // `Range` needs to check `start < end`, but thanks to our type invariant + // we can loop on the stricter `start != end`. + + self.assume_range(); + while self.start != self.end { + // SAFETY: We just checked that the range is non-empty + let i = unsafe { self.next_unchecked() }; + accum = f(accum, i)?; + } + try { accum } + } } impl DoubleEndedIterator for IndexRange { @@ -156,6 +187,30 @@ impl DoubleEndedIterator for IndexRange { let taken = self.take_suffix(n); NonZero::new(n - taken.len()).map_or(Ok(()), Err) } + + #[inline] + fn rfold<B, F: FnMut(B, usize) -> B>(mut self, init: B, f: F) -> B { + self.try_rfold(init, NeverShortCircuit::wrap_mut_2(f)).0 + } + + #[inline] + fn try_rfold<B, F, R>(&mut self, mut accum: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: Try<Output = B>, + { + // `Range` needs to check `start < end`, but thanks to our type invariant + // we can loop on the stricter `start != end`. + + self.assume_range(); + while self.start != self.end { + // SAFETY: We just checked that the range is non-empty + let i = unsafe { self.next_back_unchecked() }; + accum = f(accum, i)?; + } + try { accum } + } } impl ExactSizeIterator for IndexRange { diff --git a/library/core/src/pat.rs b/library/core/src/pat.rs index f8826096df3..91d015b1bc5 100644 --- a/library/core/src/pat.rs +++ b/library/core/src/pat.rs @@ -25,15 +25,15 @@ macro_rules! pattern_type { )] pub trait RangePattern { /// Trait version of the inherent `MIN` assoc const. - #[cfg_attr(not(bootstrap), lang = "RangeMin")] + #[lang = "RangeMin"] const MIN: Self; /// Trait version of the inherent `MIN` assoc const. - #[cfg_attr(not(bootstrap), lang = "RangeMax")] + #[lang = "RangeMax"] const MAX: Self; /// A compile-time helper to subtract 1 for exclusive ranges. - #[cfg_attr(not(bootstrap), lang = "RangeSub")] + #[lang = "RangeSub"] #[track_caller] fn sub_one(self) -> Self; } diff --git a/library/core/src/pin.rs b/library/core/src/pin.rs index bc097bf198d..9e6acf04bf7 100644 --- a/library/core/src/pin.rs +++ b/library/core/src/pin.rs @@ -931,6 +931,11 @@ use crate::{ }; use crate::{cmp, fmt}; +mod unsafe_pinned; + +#[unstable(feature = "unsafe_pinned", issue = "125735")] +pub use self::unsafe_pinned::UnsafePinned; + /// A pointer which pins its pointee in place. /// /// [`Pin`] is a wrapper around some kind of pointer `Ptr` which makes that pointer "pin" its @@ -1087,24 +1092,15 @@ use crate::{cmp, fmt}; #[rustc_pub_transparent] #[derive(Copy, Clone)] pub struct Pin<Ptr> { - // FIXME(#93176): this field is made `#[unstable] #[doc(hidden)] pub` to: - // - deter downstream users from accessing it (which would be unsound!), - // - let the `pin!` macro access it (such a macro requires using struct - // literal syntax in order to benefit from lifetime extension). - // - // However, if the `Deref` impl exposes a field with the same name as this - // field, then the two will collide, resulting in a confusing error when the - // user attempts to access the field through a `Pin<Ptr>`. Therefore, the - // name `__pointer` is designed to be unlikely to collide with any other - // field. Long-term, macro hygiene is expected to offer a more robust - // alternative, alongside `unsafe` fields. - #[unstable(feature = "unsafe_pin_internals", issue = "none")] - #[doc(hidden)] - pub __pointer: Ptr, + /// Only public for bootstrap. + #[cfg(bootstrap)] + pub pointer: Ptr, + #[cfg(not(bootstrap))] + pointer: Ptr, } // The following implementations aren't derived in order to avoid soundness -// issues. `&self.__pointer` should not be accessible to untrusted trait +// issues. `&self.pointer` should not be accessible to untrusted trait // implementations. // // See <https://internals.rust-lang.org/t/unsoundness-in-pin/11311/73> for more details. @@ -1218,7 +1214,7 @@ impl<Ptr: Deref<Target: Unpin>> Pin<Ptr> { #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] #[stable(feature = "pin_into_inner", since = "1.39.0")] pub const fn into_inner(pin: Pin<Ptr>) -> Ptr { - pin.__pointer + pin.pointer } } @@ -1355,7 +1351,7 @@ impl<Ptr: Deref> Pin<Ptr> { #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] #[stable(feature = "pin", since = "1.33.0")] pub const unsafe fn new_unchecked(pointer: Ptr) -> Pin<Ptr> { - Pin { __pointer: pointer } + Pin { pointer } } /// Gets a shared reference to the pinned value this [`Pin`] points to. @@ -1369,7 +1365,7 @@ impl<Ptr: Deref> Pin<Ptr> { #[inline(always)] pub fn as_ref(&self) -> Pin<&Ptr::Target> { // SAFETY: see documentation on this function - unsafe { Pin::new_unchecked(&*self.__pointer) } + unsafe { Pin::new_unchecked(&*self.pointer) } } } @@ -1413,7 +1409,7 @@ impl<Ptr: DerefMut> Pin<Ptr> { #[inline(always)] pub fn as_mut(&mut self) -> Pin<&mut Ptr::Target> { // SAFETY: see documentation on this function - unsafe { Pin::new_unchecked(&mut *self.__pointer) } + unsafe { Pin::new_unchecked(&mut *self.pointer) } } /// Gets `Pin<&mut T>` to the underlying pinned value from this nested `Pin`-pointer. @@ -1480,7 +1476,7 @@ impl<Ptr: DerefMut> Pin<Ptr> { where Ptr::Target: Sized, { - *(self.__pointer) = value; + *(self.pointer) = value; } } @@ -1508,7 +1504,7 @@ impl<Ptr: Deref> Pin<Ptr> { #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] #[stable(feature = "pin_into_inner", since = "1.39.0")] pub const unsafe fn into_inner_unchecked(pin: Pin<Ptr>) -> Ptr { - pin.__pointer + pin.pointer } } @@ -1534,7 +1530,7 @@ impl<'a, T: ?Sized> Pin<&'a T> { U: ?Sized, F: FnOnce(&T) -> &U, { - let pointer = &*self.__pointer; + let pointer = &*self.pointer; let new_pointer = func(pointer); // SAFETY: the safety contract for `new_unchecked` must be @@ -1564,7 +1560,7 @@ impl<'a, T: ?Sized> Pin<&'a T> { #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] #[stable(feature = "pin", since = "1.33.0")] pub const fn get_ref(self) -> &'a T { - self.__pointer + self.pointer } } @@ -1575,7 +1571,7 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] #[stable(feature = "pin", since = "1.33.0")] pub const fn into_ref(self) -> Pin<&'a T> { - Pin { __pointer: self.__pointer } + Pin { pointer: self.pointer } } /// Gets a mutable reference to the data inside of this `Pin`. @@ -1595,7 +1591,7 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { where T: Unpin, { - self.__pointer + self.pointer } /// Gets a mutable reference to the data inside of this `Pin`. @@ -1613,7 +1609,7 @@ impl<'a, T: ?Sized> Pin<&'a mut T> { #[stable(feature = "pin", since = "1.33.0")] #[rustc_const_stable(feature = "const_pin", since = "1.84.0")] pub const unsafe fn get_unchecked_mut(self) -> &'a mut T { - self.__pointer + self.pointer } /// Constructs a new pin by mapping the interior value. @@ -1700,21 +1696,21 @@ impl<Ptr: LegacyReceiver> LegacyReceiver for Pin<Ptr> {} #[stable(feature = "pin", since = "1.33.0")] impl<Ptr: fmt::Debug> fmt::Debug for Pin<Ptr> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(&self.__pointer, f) + fmt::Debug::fmt(&self.pointer, f) } } #[stable(feature = "pin", since = "1.33.0")] impl<Ptr: fmt::Display> fmt::Display for Pin<Ptr> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.__pointer, f) + fmt::Display::fmt(&self.pointer, f) } } #[stable(feature = "pin", since = "1.33.0")] impl<Ptr: fmt::Pointer> fmt::Pointer for Pin<Ptr> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Pointer::fmt(&self.__pointer, f) + fmt::Pointer::fmt(&self.pointer, f) } } @@ -1940,80 +1936,22 @@ unsafe impl<T: ?Sized> PinCoerceUnsized for *mut T {} /// constructor. /// /// [`Box::pin`]: ../../std/boxed/struct.Box.html#method.pin +#[cfg(not(bootstrap))] +#[stable(feature = "pin_macro", since = "1.68.0")] +#[rustc_macro_transparency = "semitransparent"] +#[allow_internal_unstable(super_let)] +pub macro pin($value:expr $(,)?) { + { + super let mut pinned = $value; + // SAFETY: The value is pinned: it is the local above which cannot be named outside this macro. + unsafe { $crate::pin::Pin::new_unchecked(&mut pinned) } + } +} + +/// Only for bootstrap. +#[cfg(bootstrap)] #[stable(feature = "pin_macro", since = "1.68.0")] #[rustc_macro_transparency = "semitransparent"] -#[allow_internal_unstable(unsafe_pin_internals)] -#[cfg_attr(not(bootstrap), rustc_macro_edition_2021)] pub macro pin($value:expr $(,)?) { - // This is `Pin::new_unchecked(&mut { $value })`, so, for starters, let's - // review such a hypothetical macro (that any user-code could define): - // - // ```rust - // macro_rules! pin {( $value:expr ) => ( - // match &mut { $value } { at_value => unsafe { // Do not wrap `$value` in an `unsafe` block. - // $crate::pin::Pin::<&mut _>::new_unchecked(at_value) - // }} - // )} - // ``` - // - // Safety: - // - `type P = &mut _`. There are thus no pathological `Deref{,Mut}` impls - // that would break `Pin`'s invariants. - // - `{ $value }` is braced, making it a _block expression_, thus **moving** - // the given `$value`, and making it _become an **anonymous** temporary_. - // By virtue of being anonymous, it can no longer be accessed, thus - // preventing any attempts to `mem::replace` it or `mem::forget` it, _etc._ - // - // This gives us a `pin!` definition that is sound, and which works, but only - // in certain scenarios: - // - If the `pin!(value)` expression is _directly_ fed to a function call: - // `let poll = pin!(fut).poll(cx);` - // - If the `pin!(value)` expression is part of a scrutinee: - // ```rust - // match pin!(fut) { pinned_fut => { - // pinned_fut.as_mut().poll(...); - // pinned_fut.as_mut().poll(...); - // }} // <- `fut` is dropped here. - // ``` - // Alas, it doesn't work for the more straight-forward use-case: `let` bindings. - // ```rust - // let pinned_fut = pin!(fut); // <- temporary value is freed at the end of this statement - // pinned_fut.poll(...) // error[E0716]: temporary value dropped while borrowed - // // note: consider using a `let` binding to create a longer lived value - // ``` - // - Issues such as this one are the ones motivating https://github.com/rust-lang/rfcs/pull/66 - // - // This makes such a macro incredibly unergonomic in practice, and the reason most macros - // out there had to take the path of being a statement/binding macro (_e.g._, `pin!(future);`) - // instead of featuring the more intuitive ergonomics of an expression macro. - // - // Luckily, there is a way to avoid the problem. Indeed, the problem stems from the fact that a - // temporary is dropped at the end of its enclosing statement when it is part of the parameters - // given to function call, which has precisely been the case with our `Pin::new_unchecked()`! - // For instance, - // ```rust - // let p = Pin::new_unchecked(&mut <temporary>); - // ``` - // becomes: - // ```rust - // let p = { let mut anon = <temporary>; &mut anon }; - // ``` - // - // However, when using a literal braced struct to construct the value, references to temporaries - // can then be taken. This makes Rust change the lifespan of such temporaries so that they are, - // instead, dropped _at the end of the enscoping block_. - // For instance, - // ```rust - // let p = Pin { __pointer: &mut <temporary> }; - // ``` - // becomes: - // ```rust - // let mut anon = <temporary>; - // let p = Pin { __pointer: &mut anon }; - // ``` - // which is *exactly* what we want. - // - // See https://doc.rust-lang.org/1.58.1/reference/destructors.html#temporary-lifetime-extension - // for more info. - $crate::pin::Pin::<&mut _> { __pointer: &mut { $value } } + $crate::pin::Pin::<&mut _> { pointer: &mut { $value } } } diff --git a/library/core/src/pin/unsafe_pinned.rs b/library/core/src/pin/unsafe_pinned.rs new file mode 100644 index 00000000000..5fb628c8adb --- /dev/null +++ b/library/core/src/pin/unsafe_pinned.rs @@ -0,0 +1,197 @@ +use crate::marker::{PointerLike, Unpin}; +use crate::ops::{CoerceUnsized, DispatchFromDyn}; +use crate::pin::Pin; +use crate::{fmt, ptr}; + +/// This type provides a way to opt-out of typical aliasing rules; +/// specifically, `&mut UnsafePinned<T>` is not guaranteed to be a unique pointer. +/// +/// However, even if you define your type like `pub struct Wrapper(UnsafePinned<...>)`, it is still +/// very risky to have an `&mut Wrapper` that aliases anything else. Many functions that work +/// generically on `&mut T` assume that the memory that stores `T` is uniquely owned (such as +/// `mem::swap`). In other words, while having aliasing with `&mut Wrapper` is not immediate +/// Undefined Behavior, it is still unsound to expose such a mutable reference to code you do not +/// control! Techniques such as pinning via [`Pin`] are needed to ensure soundness. +/// +/// Similar to [`UnsafeCell`](crate::cell::UnsafeCell), `UnsafePinned` will not usually show up in +/// the public API of a library. It is an internal implementation detail of libraries that need to +/// support aliasing mutable references. +/// +/// Further note that this does *not* lift the requirement that shared references must be read-only! +/// Use `UnsafeCell` for that. +/// +/// This type blocks niches the same way `UnsafeCell` does. +#[cfg_attr(not(bootstrap), lang = "unsafe_pinned")] +#[repr(transparent)] +#[unstable(feature = "unsafe_pinned", issue = "125735")] +pub struct UnsafePinned<T: ?Sized> { + value: T, +} + +/// When this type is used, that almost certainly means safe APIs need to use pinning to avoid the +/// aliases from becoming invalidated. Therefore let's mark this as `!Unpin`. You can always opt +/// back in to `Unpin` with an `impl` block, provided your API is still sound while unpinned. +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: ?Sized> !Unpin for UnsafePinned<T> {} + +/// The type is `Copy` when `T` is to avoid people assuming that `Copy` implies there is no +/// `UnsafePinned` anywhere. (This is an issue with `UnsafeCell`: people use `Copy` bounds to mean +/// `Freeze`.) Given that there is no `unsafe impl Copy for ...`, this is also the option that +/// leaves the user more choices (as they can always wrap this in a `!Copy` type). +// FIXME(unsafe_pinned): this may be unsound or a footgun? +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: Copy> Copy for UnsafePinned<T> {} + +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: Copy> Clone for UnsafePinned<T> { + fn clone(&self) -> Self { + *self + } +} + +// `Send` and `Sync` are inherited from `T`. This is similar to `SyncUnsafeCell`, since +// we eventually concluded that `UnsafeCell` implicitly making things `!Sync` is sometimes +// unergonomic. A type that needs to be `!Send`/`!Sync` should really have an explicit +// opt-out itself, e.g. via an `PhantomData<*mut T>` or (one day) via `impl !Send`/`impl !Sync`. + +impl<T> UnsafePinned<T> { + /// Constructs a new instance of `UnsafePinned` which will wrap the specified value. + /// + /// All access to the inner value through `&UnsafePinned<T>` or `&mut UnsafePinned<T>` or + /// `Pin<&mut UnsafePinned<T>>` requires `unsafe` code. + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn new(value: T) -> Self { + UnsafePinned { value } + } + + /// Unwraps the value, consuming this `UnsafePinned`. + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + #[rustc_allow_const_fn_unstable(const_precise_live_drops)] + pub const fn into_inner(self) -> T { + self.value + } +} + +impl<T: ?Sized> UnsafePinned<T> { + /// Get read-write access to the contents of a pinned `UnsafePinned`. + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn get_mut_pinned(self: Pin<&mut Self>) -> *mut T { + // SAFETY: we're not using `get_unchecked_mut` to unpin anything + unsafe { self.get_unchecked_mut() }.get_mut_unchecked() + } + + /// Get read-write access to the contents of an `UnsafePinned`. + /// + /// You should usually be using `get_mut_pinned` instead to explicitly track the fact that this + /// memory is "pinned" due to there being aliases. + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn get_mut_unchecked(&mut self) -> *mut T { + ptr::from_mut(self) as *mut T + } + + /// Get read-only access to the contents of a shared `UnsafePinned`. + /// + /// Note that `&UnsafePinned<T>` is read-only if `&T` is read-only. This means that if there is + /// mutation of the `T`, future reads from the `*const T` returned here are UB! Use + /// [`UnsafeCell`] if you also need interior mutability. + /// + /// [`UnsafeCell`]: crate::cell::UnsafeCell + /// + /// ```rust,no_run + /// #![feature(unsafe_pinned)] + /// use std::pin::UnsafePinned; + /// + /// unsafe { + /// let mut x = UnsafePinned::new(0); + /// let ptr = x.get(); // read-only pointer, assumes immutability + /// x.get_mut_unchecked().write(1); + /// ptr.read(); // UB! + /// } + /// ``` + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn get(&self) -> *const T { + ptr::from_ref(self) as *const T + } + + /// Gets an immutable pointer to the wrapped value. + /// + /// The difference from [`get`] is that this function accepts a raw pointer, which is useful to + /// avoid the creation of temporary references. + /// + /// [`get`]: UnsafePinned::get + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn raw_get(this: *const Self) -> *const T { + this as *const T + } + + /// Gets a mutable pointer to the wrapped value. + /// + /// The difference from [`get_mut_pinned`] and [`get_mut_unchecked`] is that this function + /// accepts a raw pointer, which is useful to avoid the creation of temporary references. + /// + /// [`get_mut_pinned`]: UnsafePinned::get_mut_pinned + /// [`get_mut_unchecked`]: UnsafePinned::get_mut_unchecked + #[inline(always)] + #[must_use] + #[unstable(feature = "unsafe_pinned", issue = "125735")] + pub const fn raw_get_mut(this: *mut Self) -> *mut T { + this as *mut T + } +} + +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: Default> Default for UnsafePinned<T> { + /// Creates an `UnsafePinned`, with the `Default` value for T. + fn default() -> Self { + UnsafePinned::new(T::default()) + } +} + +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T> From<T> for UnsafePinned<T> { + /// Creates a new `UnsafePinned<T>` containing the given value. + fn from(value: T) -> Self { + UnsafePinned::new(value) + } +} + +#[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: ?Sized> fmt::Debug for UnsafePinned<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("UnsafePinned").finish_non_exhaustive() + } +} + +#[unstable(feature = "coerce_unsized", issue = "18598")] +// #[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: CoerceUnsized<U>, U> CoerceUnsized<UnsafePinned<U>> for UnsafePinned<T> {} + +// Allow types that wrap `UnsafePinned` to also implement `DispatchFromDyn` +// and become dyn-compatible method receivers. +// Note that currently `UnsafePinned` itself cannot be a method receiver +// because it does not implement Deref. +// In other words: +// `self: UnsafePinned<&Self>` won't work +// `self: UnsafePinned<Self>` becomes possible +// FIXME(unsafe_pinned) this logic is copied from UnsafeCell, is it still sound? +#[unstable(feature = "dispatch_from_dyn", issue = "none")] +// #[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: DispatchFromDyn<U>, U> DispatchFromDyn<UnsafePinned<U>> for UnsafePinned<T> {} + +#[unstable(feature = "pointer_like_trait", issue = "none")] +// #[unstable(feature = "unsafe_pinned", issue = "125735")] +impl<T: PointerLike> PointerLike for UnsafePinned<T> {} + +// FIXME(unsafe_pinned): impl PinCoerceUnsized for UnsafePinned<T>? diff --git a/library/core/src/prelude/v1.rs b/library/core/src/prelude/v1.rs index c5975c03050..8f1b5275871 100644 --- a/library/core/src/prelude/v1.rs +++ b/library/core/src/prelude/v1.rs @@ -117,5 +117,4 @@ pub use crate::macros::builtin::deref; issue = "63063", reason = "`type_alias_impl_trait` has open design concerns" )] -#[cfg(not(bootstrap))] pub use crate::macros::builtin::define_opaque; diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 71a84aff246..0854e31c199 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -764,8 +764,8 @@ impl<T: ?Sized> *const T { /// // This would be incorrect, as the pointers are not correctly ordered: /// // ptr1.offset_from_unsigned(ptr2) /// ``` - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset_from_unsigned(self, origin: *const T) -> usize @@ -809,8 +809,8 @@ impl<T: ?Sized> *const T { /// /// For non-`Sized` pointees this operation considers only the data pointers, /// ignoring the metadata. - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn byte_offset_from_unsigned<U: ?Sized>(self, origin: *const U) -> usize { diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index ea53da78d3b..445c789a7de 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -278,7 +278,7 @@ //! ### Using Strict Provenance //! //! Most code needs no changes to conform to strict provenance, as the only really concerning -//! operation is casts from usize to a pointer. For code which *does* cast a `usize` to a pointer, +//! operation is casts from `usize` to a pointer. For code which *does* cast a `usize` to a pointer, //! the scope of the change depends on exactly what you're doing. //! //! In general, you just need to make sure that if you want to convert a `usize` address to a @@ -398,6 +398,7 @@ use crate::cmp::Ordering; use crate::intrinsics::const_eval_select; use crate::marker::FnPtr; use crate::mem::{self, MaybeUninit, SizedTypeProperties}; +use crate::num::NonZero; use crate::{fmt, hash, intrinsics, ub_checks}; mod alignment; @@ -1094,51 +1095,25 @@ pub const unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) { // are pointers inside `T` we will copy them in one go rather than trying to copy a part // of a pointer (which would not work). // SAFETY: Same preconditions as this function - unsafe { swap_nonoverlapping_simple_untyped(x, y, count) } + unsafe { swap_nonoverlapping_const(x, y, count) } } else { - macro_rules! attempt_swap_as_chunks { - ($ChunkTy:ty) => { - if align_of::<T>() >= align_of::<$ChunkTy>() - && size_of::<T>() % size_of::<$ChunkTy>() == 0 - { - let x: *mut $ChunkTy = x.cast(); - let y: *mut $ChunkTy = y.cast(); - let count = count * (size_of::<T>() / size_of::<$ChunkTy>()); - // SAFETY: these are the same bytes that the caller promised were - // ok, just typed as `MaybeUninit<ChunkTy>`s instead of as `T`s. - // The `if` condition above ensures that we're not violating - // alignment requirements, and that the division is exact so - // that we don't lose any bytes off the end. - return unsafe { swap_nonoverlapping_simple_untyped(x, y, count) }; - } - }; + // Going though a slice here helps codegen know the size fits in `isize` + let slice = slice_from_raw_parts_mut(x, count); + // SAFETY: This is all readable from the pointer, meaning it's one + // allocated object, and thus cannot be more than isize::MAX bytes. + let bytes = unsafe { mem::size_of_val_raw::<[T]>(slice) }; + if let Some(bytes) = NonZero::new(bytes) { + // SAFETY: These are the same ranges, just expressed in a different + // type, so they're still non-overlapping. + unsafe { swap_nonoverlapping_bytes(x.cast(), y.cast(), bytes) }; } - - // Split up the slice into small power-of-two-sized chunks that LLVM is able - // to vectorize (unless it's a special type with more-than-pointer alignment, - // because we don't want to pessimize things like slices of SIMD vectors.) - if align_of::<T>() <= size_of::<usize>() - && (!size_of::<T>().is_power_of_two() - || size_of::<T>() > size_of::<usize>() * 2) - { - attempt_swap_as_chunks!(usize); - attempt_swap_as_chunks!(u8); - } - - // SAFETY: Same preconditions as this function - unsafe { swap_nonoverlapping_simple_untyped(x, y, count) } } ) } /// Same behavior and safety conditions as [`swap_nonoverlapping`] -/// -/// LLVM can vectorize this (at least it can for the power-of-two-sized types -/// `swap_nonoverlapping` tries to use) so no need to manually SIMD it. #[inline] -const unsafe fn swap_nonoverlapping_simple_untyped<T>(x: *mut T, y: *mut T, count: usize) { - let x = x.cast::<MaybeUninit<T>>(); - let y = y.cast::<MaybeUninit<T>>(); +const unsafe fn swap_nonoverlapping_const<T>(x: *mut T, y: *mut T, count: usize) { let mut i = 0; while i < count { // SAFETY: By precondition, `i` is in-bounds because it's below `n` @@ -1147,26 +1122,91 @@ const unsafe fn swap_nonoverlapping_simple_untyped<T>(x: *mut T, y: *mut T, coun // and it's distinct from `x` since the ranges are non-overlapping let y = unsafe { y.add(i) }; - // If we end up here, it's because we're using a simple type -- like - // a small power-of-two-sized thing -- or a special type with particularly - // large alignment, particularly SIMD types. - // Thus, we're fine just reading-and-writing it, as either it's small - // and that works well anyway or it's special and the type's author - // presumably wanted things to be done in the larger chunk. - // SAFETY: we're only ever given pointers that are valid to read/write, // including being aligned, and nothing here panics so it's drop-safe. unsafe { - let a: MaybeUninit<T> = read(x); - let b: MaybeUninit<T> = read(y); - write(x, b); - write(y, a); + // Note that it's critical that these use `copy_nonoverlapping`, + // rather than `read`/`write`, to avoid #134713 if T has padding. + let mut temp = MaybeUninit::<T>::uninit(); + copy_nonoverlapping(x, temp.as_mut_ptr(), 1); + copy_nonoverlapping(y, x, 1); + copy_nonoverlapping(temp.as_ptr(), y, 1); } i += 1; } } +// Don't let MIR inline this, because we really want it to keep its noalias metadata +#[rustc_no_mir_inline] +#[inline] +fn swap_chunk<const N: usize>(x: &mut MaybeUninit<[u8; N]>, y: &mut MaybeUninit<[u8; N]>) { + let a = *x; + let b = *y; + *x = b; + *y = a; +} + +#[inline] +unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, bytes: NonZero<usize>) { + // Same as `swap_nonoverlapping::<[u8; N]>`. + unsafe fn swap_nonoverlapping_chunks<const N: usize>( + x: *mut MaybeUninit<[u8; N]>, + y: *mut MaybeUninit<[u8; N]>, + chunks: NonZero<usize>, + ) { + let chunks = chunks.get(); + for i in 0..chunks { + // SAFETY: i is in [0, chunks) so the adds and dereferences are in-bounds. + unsafe { swap_chunk(&mut *x.add(i), &mut *y.add(i)) }; + } + } + + // Same as `swap_nonoverlapping_bytes`, but accepts at most 1+2+4=7 bytes + #[inline] + unsafe fn swap_nonoverlapping_short(x: *mut u8, y: *mut u8, bytes: NonZero<usize>) { + // Tail handling for auto-vectorized code sometimes has element-at-a-time behaviour, + // see <https://github.com/rust-lang/rust/issues/134946>. + // By swapping as different sizes, rather than as a loop over bytes, + // we make sure not to end up with, say, seven byte-at-a-time copies. + + let bytes = bytes.get(); + let mut i = 0; + macro_rules! swap_prefix { + ($($n:literal)+) => {$( + if (bytes & $n) != 0 { + // SAFETY: `i` can only have the same bits set as those in bytes, + // so these `add`s are in-bounds of `bytes`. But the bit for + // `$n` hasn't been set yet, so the `$n` bytes that `swap_chunk` + // will read and write are within the usable range. + unsafe { swap_chunk::<$n>(&mut*x.add(i).cast(), &mut*y.add(i).cast()) }; + i |= $n; + } + )+}; + } + swap_prefix!(4 2 1); + debug_assert_eq!(i, bytes); + } + + const CHUNK_SIZE: usize = size_of::<*const ()>(); + let bytes = bytes.get(); + + let chunks = bytes / CHUNK_SIZE; + let tail = bytes % CHUNK_SIZE; + if let Some(chunks) = NonZero::new(chunks) { + // SAFETY: this is bytes/CHUNK_SIZE*CHUNK_SIZE bytes, which is <= bytes, + // so it's within the range of our non-overlapping bytes. + unsafe { swap_nonoverlapping_chunks::<CHUNK_SIZE>(x.cast(), y.cast(), chunks) }; + } + if let Some(tail) = NonZero::new(tail) { + const { assert!(CHUNK_SIZE <= 8) }; + let delta = chunks * CHUNK_SIZE; + // SAFETY: the tail length is below CHUNK SIZE because of the remainder, + // and CHUNK_SIZE is at most 8 by the const assert, so tail <= 7 + unsafe { swap_nonoverlapping_short(x.add(delta), y.add(delta), tail) }; + } +} + /// Moves `src` into the pointed `dst`, returning the previous `dst` value. /// /// Neither value is dropped. diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 9e62beb8049..e29774963db 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -937,8 +937,8 @@ impl<T: ?Sized> *mut T { /// /// // This would be incorrect, as the pointers are not correctly ordered: /// // ptr1.offset_from(ptr2) - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn offset_from_unsigned(self, origin: *const T) -> usize @@ -959,8 +959,8 @@ impl<T: ?Sized> *mut T { /// /// For non-`Sized` pointees this operation considers only the data pointers, /// ignoring the metadata. - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub const unsafe fn byte_offset_from_unsigned<U: ?Sized>(self, origin: *mut U) -> usize { diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index e019aafad39..68b8f0c3e4e 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -900,8 +900,8 @@ impl<T: ?Sized> NonNull<T> { /// ``` #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] pub const unsafe fn offset_from_unsigned(self, subtracted: NonNull<T>) -> usize where T: Sized, @@ -922,8 +922,8 @@ impl<T: ?Sized> NonNull<T> { /// ignoring the metadata. #[inline(always)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - #[stable(feature = "ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "ptr_sub_ptr", since = "1.87.0")] + #[rustc_const_stable(feature = "const_ptr_sub_ptr", since = "1.87.0")] pub const unsafe fn byte_offset_from_unsigned<U: ?Sized>(self, origin: NonNull<U>) -> usize { // SAFETY: the caller must uphold the safety contract for `byte_sub_ptr`. unsafe { self.as_ptr().byte_offset_from_unsigned(origin.as_ptr()) } diff --git a/library/core/src/slice/cmp.rs b/library/core/src/slice/cmp.rs index da85f42926e..5ce72b46eee 100644 --- a/library/core/src/slice/cmp.rs +++ b/library/core/src/slice/cmp.rs @@ -5,6 +5,7 @@ use crate::ascii; use crate::cmp::{self, BytewiseEq, Ordering}; use crate::intrinsics::compare_bytes; use crate::num::NonZero; +use crate::ops::ControlFlow; #[stable(feature = "rust1", since = "1.0.0")] impl<T, U> PartialEq<[U]> for [T] @@ -31,12 +32,64 @@ impl<T: Ord> Ord for [T] { } } +#[inline] +fn as_underlying(x: ControlFlow<bool>) -> u8 { + // SAFETY: This will only compile if `bool` and `ControlFlow<bool>` have the same + // size (which isn't guaranteed but this is libcore). Because they have the same + // size, it's a niched implementation, which in one byte means there can't be + // any uninitialized memory. The callers then only check for `0` or `1` from this, + // which must necessarily match the `Break` variant, and we're fine no matter + // what ends up getting picked as the value representing `Continue(())`. + unsafe { crate::mem::transmute(x) } +} + /// Implements comparison of slices [lexicographically](Ord#lexicographical-comparison). #[stable(feature = "rust1", since = "1.0.0")] impl<T: PartialOrd> PartialOrd for [T] { + #[inline] fn partial_cmp(&self, other: &[T]) -> Option<Ordering> { SlicePartialOrd::partial_compare(self, other) } + #[inline] + fn lt(&self, other: &Self) -> bool { + // This is certainly not the obvious way to implement these methods. + // Unfortunately, using anything that looks at the discriminant means that + // LLVM sees a check for `2` (aka `ControlFlow<bool>::Continue(())`) and + // gets very distracted by that, ending up generating extraneous code. + // This should be changed to something simpler once either LLVM is smarter, + // see <https://github.com/llvm/llvm-project/issues/132678>, or we generate + // niche discriminant checks in a way that doesn't trigger it. + + as_underlying(self.__chaining_lt(other)) == 1 + } + #[inline] + fn le(&self, other: &Self) -> bool { + as_underlying(self.__chaining_le(other)) != 0 + } + #[inline] + fn gt(&self, other: &Self) -> bool { + as_underlying(self.__chaining_gt(other)) == 1 + } + #[inline] + fn ge(&self, other: &Self) -> bool { + as_underlying(self.__chaining_ge(other)) != 0 + } + #[inline] + fn __chaining_lt(&self, other: &Self) -> ControlFlow<bool> { + SliceChain::chaining_lt(self, other) + } + #[inline] + fn __chaining_le(&self, other: &Self) -> ControlFlow<bool> { + SliceChain::chaining_le(self, other) + } + #[inline] + fn __chaining_gt(&self, other: &Self) -> ControlFlow<bool> { + SliceChain::chaining_gt(self, other) + } + #[inline] + fn __chaining_ge(&self, other: &Self) -> ControlFlow<bool> { + SliceChain::chaining_ge(self, other) + } } #[doc(hidden)] @@ -99,24 +152,63 @@ trait SlicePartialOrd: Sized { fn partial_compare(left: &[Self], right: &[Self]) -> Option<Ordering>; } +#[doc(hidden)] +// intermediate trait for specialization of slice's PartialOrd chaining methods +trait SliceChain: Sized { + fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow<bool>; + fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow<bool>; + fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow<bool>; + fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow<bool>; +} + +type AlwaysBreak<B> = ControlFlow<B, crate::convert::Infallible>; + impl<A: PartialOrd> SlicePartialOrd for A { default fn partial_compare(left: &[A], right: &[A]) -> Option<Ordering> { - let l = cmp::min(left.len(), right.len()); - - // Slice to the loop iteration range to enable bound check - // elimination in the compiler - let lhs = &left[..l]; - let rhs = &right[..l]; + let elem_chain = |a, b| match PartialOrd::partial_cmp(a, b) { + Some(Ordering::Equal) => ControlFlow::Continue(()), + non_eq => ControlFlow::Break(non_eq), + }; + let len_chain = |a: &_, b: &_| ControlFlow::Break(usize::partial_cmp(a, b)); + let AlwaysBreak::Break(b) = chaining_impl(left, right, elem_chain, len_chain); + b + } +} - for i in 0..l { - match lhs[i].partial_cmp(&rhs[i]) { - Some(Ordering::Equal) => (), - non_eq => return non_eq, - } - } +impl<A: PartialOrd> SliceChain for A { + default fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + chaining_impl(left, right, PartialOrd::__chaining_lt, usize::__chaining_lt) + } + default fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + chaining_impl(left, right, PartialOrd::__chaining_le, usize::__chaining_le) + } + default fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + chaining_impl(left, right, PartialOrd::__chaining_gt, usize::__chaining_gt) + } + default fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + chaining_impl(left, right, PartialOrd::__chaining_ge, usize::__chaining_ge) + } +} - left.len().partial_cmp(&right.len()) +#[inline] +fn chaining_impl<'l, 'r, A: PartialOrd, B, C>( + left: &'l [A], + right: &'r [A], + elem_chain: impl Fn(&'l A, &'r A) -> ControlFlow<B>, + len_chain: impl for<'a> FnOnce(&'a usize, &'a usize) -> ControlFlow<B, C>, +) -> ControlFlow<B, C> { + let l = cmp::min(left.len(), right.len()); + + // Slice to the loop iteration range to enable bound check + // elimination in the compiler + let lhs = &left[..l]; + let rhs = &right[..l]; + + for i in 0..l { + elem_chain(&lhs[i], &rhs[i])?; } + + len_chain(&left.len(), &right.len()) } // This is the impl that we would like to have. Unfortunately it's not sound. @@ -165,21 +257,13 @@ trait SliceOrd: Sized { impl<A: Ord> SliceOrd for A { default fn compare(left: &[Self], right: &[Self]) -> Ordering { - let l = cmp::min(left.len(), right.len()); - - // Slice to the loop iteration range to enable bound check - // elimination in the compiler - let lhs = &left[..l]; - let rhs = &right[..l]; - - for i in 0..l { - match lhs[i].cmp(&rhs[i]) { - Ordering::Equal => (), - non_eq => return non_eq, - } - } - - left.len().cmp(&right.len()) + let elem_chain = |a, b| match Ord::cmp(a, b) { + Ordering::Equal => ControlFlow::Continue(()), + non_eq => ControlFlow::Break(non_eq), + }; + let len_chain = |a: &_, b: &_| ControlFlow::Break(usize::cmp(a, b)); + let AlwaysBreak::Break(b) = chaining_impl(left, right, elem_chain, len_chain); + b } } @@ -191,7 +275,7 @@ impl<A: Ord> SliceOrd for A { /// * For every `x` and `y` of this type, `Ord(x, y)` must return the same /// value as `Ord::cmp(transmute::<_, u8>(x), transmute::<_, u8>(y))`. #[rustc_specialization_trait] -unsafe trait UnsignedBytewiseOrd {} +unsafe trait UnsignedBytewiseOrd: Ord {} unsafe impl UnsignedBytewiseOrd for bool {} unsafe impl UnsignedBytewiseOrd for u8 {} @@ -225,6 +309,38 @@ impl<A: Ord + UnsignedBytewiseOrd> SliceOrd for A { } } +// Don't generate our own chaining loops for `memcmp`-able things either. +impl<A: PartialOrd + UnsignedBytewiseOrd> SliceChain for A { + #[inline] + fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + match SliceOrd::compare(left, right) { + Ordering::Equal => ControlFlow::Continue(()), + ne => ControlFlow::Break(ne.is_lt()), + } + } + #[inline] + fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + match SliceOrd::compare(left, right) { + Ordering::Equal => ControlFlow::Continue(()), + ne => ControlFlow::Break(ne.is_le()), + } + } + #[inline] + fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + match SliceOrd::compare(left, right) { + Ordering::Equal => ControlFlow::Continue(()), + ne => ControlFlow::Break(ne.is_gt()), + } + } + #[inline] + fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow<bool> { + match SliceOrd::compare(left, right) { + Ordering::Equal => ControlFlow::Continue(()), + ne => ControlFlow::Break(ne.is_ge()), + } + } +} + pub(super) trait SliceContains: Sized { fn slice_contains(&self, x: &[Self]) -> bool; } diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs index c7e8d5f989d..7b2a3fac38a 100644 --- a/library/core/src/slice/mod.rs +++ b/library/core/src/slice/mod.rs @@ -1287,7 +1287,6 @@ impl<T> [T] { /// // let chunks: &[[_; 0]] = slice.as_chunks_unchecked() // Zero-length chunks are never allowed /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[must_use] pub const unsafe fn as_chunks_unchecked<const N: usize>(&self) -> &[[T; N]] { @@ -1333,7 +1332,6 @@ impl<T> [T] { /// assert_eq!(chunks, &[['R', 'u'], ['s', 't']]); /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[track_caller] #[must_use] @@ -1368,7 +1366,6 @@ impl<T> [T] { /// assert_eq!(chunks, &[['o', 'r'], ['e', 'm']]); /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[track_caller] #[must_use] @@ -1448,7 +1445,6 @@ impl<T> [T] { /// // let chunks: &[[_; 0]] = slice.as_chunks_unchecked_mut() // Zero-length chunks are never allowed /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[must_use] pub const unsafe fn as_chunks_unchecked_mut<const N: usize>(&mut self) -> &mut [[T; N]] { @@ -1489,7 +1485,6 @@ impl<T> [T] { /// assert_eq!(v, &[1, 1, 2, 2, 9]); /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[track_caller] #[must_use] @@ -1530,7 +1525,6 @@ impl<T> [T] { /// assert_eq!(v, &[9, 1, 1, 2, 2]); /// ``` #[unstable(feature = "slice_as_chunks", issue = "74985")] - #[rustc_const_unstable(feature = "slice_as_chunks", issue = "74985")] #[inline] #[track_caller] #[must_use] @@ -2828,7 +2822,7 @@ impl<T> [T] { // Binary search interacts poorly with branch prediction, so force // the compiler to use conditional moves if supported by the target // architecture. - base = (cmp == Greater).select_unpredictable(base, mid); + base = hint::select_unpredictable(cmp == Greater, base, mid); // This is imprecise in the case where `size` is odd and the // comparison returns Greater: the mid element still gets included @@ -3721,7 +3715,7 @@ impl<T> [T] { #[doc(alias = "memcpy")] #[inline] #[stable(feature = "copy_from_slice", since = "1.9.0")] - #[rustc_const_stable(feature = "const_copy_from_slice", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_copy_from_slice", since = "1.87.0")] #[track_caller] pub const fn copy_from_slice(&mut self, src: &[T]) where @@ -4331,7 +4325,7 @@ impl<T> [T] { /// ``` #[inline] #[must_use = "method does not modify the slice if the range is out of bounds"] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] pub fn split_off<'a, R: OneSidedRange<usize>>( self: &mut &'a Self, range: R, @@ -4397,7 +4391,7 @@ impl<T> [T] { /// ``` #[inline] #[must_use = "method does not modify the slice if the range is out of bounds"] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] pub fn split_off_mut<'a, R: OneSidedRange<usize>>( self: &mut &'a mut Self, range: R, @@ -4434,7 +4428,7 @@ impl<T> [T] { /// assert_eq!(first, &'a'); /// ``` #[inline] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] #[rustc_const_unstable(feature = "const_split_off_first_last", issue = "138539")] pub const fn split_off_first<'a>(self: &mut &'a Self) -> Option<&'a T> { // FIXME(const-hack): Use `?` when available in const instead of `let-else`. @@ -4459,7 +4453,7 @@ impl<T> [T] { /// assert_eq!(first, &'d'); /// ``` #[inline] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] #[rustc_const_unstable(feature = "const_split_off_first_last", issue = "138539")] pub const fn split_off_first_mut<'a>(self: &mut &'a mut Self) -> Option<&'a mut T> { // FIXME(const-hack): Use `mem::take` and `?` when available in const. @@ -4484,7 +4478,7 @@ impl<T> [T] { /// assert_eq!(last, &'c'); /// ``` #[inline] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] #[rustc_const_unstable(feature = "const_split_off_first_last", issue = "138539")] pub const fn split_off_last<'a>(self: &mut &'a Self) -> Option<&'a T> { // FIXME(const-hack): Use `?` when available in const instead of `let-else`. @@ -4509,7 +4503,7 @@ impl<T> [T] { /// assert_eq!(last, &'d'); /// ``` #[inline] - #[stable(feature = "slice_take", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "slice_take", since = "1.87.0")] #[rustc_const_unstable(feature = "const_split_off_first_last", issue = "138539")] pub const fn split_off_last_mut<'a>(self: &mut &'a mut Self) -> Option<&'a mut T> { // FIXME(const-hack): Use `mem::take` and `?` when available in const. @@ -4841,7 +4835,7 @@ impl<T, const N: usize> [[T; N]] { /// assert!(empty_slice_of_arrays.as_flattened().is_empty()); /// ``` #[stable(feature = "slice_flatten", since = "1.80.0")] - #[rustc_const_stable(feature = "const_slice_flatten", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_slice_flatten", since = "1.87.0")] pub const fn as_flattened(&self) -> &[T] { let len = if T::IS_ZST { self.len().checked_mul(N).expect("slice len overflow") @@ -4878,7 +4872,7 @@ impl<T, const N: usize> [[T; N]] { /// assert_eq!(array, [[6, 7, 8], [9, 10, 11], [12, 13, 14]]); /// ``` #[stable(feature = "slice_flatten", since = "1.80.0")] - #[rustc_const_stable(feature = "const_slice_flatten", since = "CURRENT_RUSTC_VERSION")] + #[rustc_const_stable(feature = "const_slice_flatten", since = "1.87.0")] pub const fn as_flattened_mut(&mut self) -> &mut [T] { let len = if T::IS_ZST { self.len().checked_mul(N).expect("slice len overflow") diff --git a/library/core/src/slice/sort/shared/smallsort.rs b/library/core/src/slice/sort/shared/smallsort.rs index 95f196a40d0..4280f7570db 100644 --- a/library/core/src/slice/sort/shared/smallsort.rs +++ b/library/core/src/slice/sort/shared/smallsort.rs @@ -2,7 +2,7 @@ use crate::mem::{self, ManuallyDrop, MaybeUninit}; use crate::slice::sort::shared::FreezeMarker; -use crate::{intrinsics, ptr, slice}; +use crate::{hint, intrinsics, ptr, slice}; // It's important to differentiate between SMALL_SORT_THRESHOLD performance for // small slices and small-sort performance sorting small sub-slices as part of @@ -408,8 +408,8 @@ where // } // The goal is to generate cmov instructions here. - let v_a_swap = should_swap.select_unpredictable(v_b, v_a); - let v_b_swap = should_swap.select_unpredictable(v_a, v_b); + let v_a_swap = hint::select_unpredictable(should_swap, v_b, v_a); + let v_b_swap = hint::select_unpredictable(should_swap, v_a, v_b); let v_b_swap_tmp = ManuallyDrop::new(ptr::read(v_b_swap)); ptr::copy(v_a_swap, v_a, 1); @@ -640,15 +640,15 @@ pub unsafe fn sort4_stable<T, F: FnMut(&T, &T) -> bool>( // 1, 1 | c b a d let c3 = is_less(&*c, &*a); let c4 = is_less(&*d, &*b); - let min = c3.select_unpredictable(c, a); - let max = c4.select_unpredictable(b, d); - let unknown_left = c3.select_unpredictable(a, c4.select_unpredictable(c, b)); - let unknown_right = c4.select_unpredictable(d, c3.select_unpredictable(b, c)); + let min = hint::select_unpredictable(c3, c, a); + let max = hint::select_unpredictable(c4, b, d); + let unknown_left = hint::select_unpredictable(c3, a, hint::select_unpredictable(c4, c, b)); + let unknown_right = hint::select_unpredictable(c4, d, hint::select_unpredictable(c3, b, c)); // Sort the last two unknown elements. let c5 = is_less(&*unknown_right, &*unknown_left); - let lo = c5.select_unpredictable(unknown_right, unknown_left); - let hi = c5.select_unpredictable(unknown_left, unknown_right); + let lo = hint::select_unpredictable(c5, unknown_right, unknown_left); + let hi = hint::select_unpredictable(c5, unknown_left, unknown_right); ptr::copy_nonoverlapping(min, dst, 1); ptr::copy_nonoverlapping(lo, dst.add(1), 1); diff --git a/library/core/src/str/converts.rs b/library/core/src/str/converts.rs index 1276d9014f0..37854a4da64 100644 --- a/library/core/src/str/converts.rs +++ b/library/core/src/str/converts.rs @@ -126,7 +126,7 @@ pub const fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> { /// See the docs for [`Utf8Error`] for more details on the kinds of /// errors that can be returned. #[stable(feature = "str_mut_extras", since = "1.20.0")] -#[rustc_const_stable(feature = "const_str_from_utf8", since = "CURRENT_RUSTC_VERSION")] +#[rustc_const_stable(feature = "const_str_from_utf8", since = "1.87.0")] #[rustc_diagnostic_item = "str_from_utf8_mut"] pub const fn from_utf8_mut(v: &mut [u8]) -> Result<&mut str, Utf8Error> { // FIXME(const-hack): This should use `?` again, once it's `const` diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs index 5cc08f8a71a..79b4953fcc1 100644 --- a/library/core/src/str/mod.rs +++ b/library/core/src/str/mod.rs @@ -230,8 +230,8 @@ impl str { /// /// assert_eq!("💖", sparkle_heart); /// ``` - #[stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "inherent_str_constructors", since = "1.87.0")] + #[rustc_const_stable(feature = "inherent_str_constructors", since = "1.87.0")] #[rustc_diagnostic_item = "str_inherent_from_utf8"] pub const fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> { converts::from_utf8(v) @@ -263,8 +263,8 @@ impl str { /// ``` /// See the docs for [`Utf8Error`] for more details on the kinds of /// errors that can be returned. - #[stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "const_str_from_utf8", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "inherent_str_constructors", since = "1.87.0")] + #[rustc_const_stable(feature = "const_str_from_utf8", since = "1.87.0")] #[rustc_diagnostic_item = "str_inherent_from_utf8_mut"] pub const fn from_utf8_mut(v: &mut [u8]) -> Result<&mut str, Utf8Error> { converts::from_utf8_mut(v) @@ -295,8 +295,8 @@ impl str { /// ``` #[inline] #[must_use] - #[stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "inherent_str_constructors", since = "1.87.0")] + #[rustc_const_stable(feature = "inherent_str_constructors", since = "1.87.0")] #[rustc_diagnostic_item = "str_inherent_from_utf8_unchecked"] pub const unsafe fn from_utf8_unchecked(v: &[u8]) -> &str { // SAFETY: converts::from_utf8_unchecked has the same safety requirements as this function. @@ -320,8 +320,8 @@ impl str { /// ``` #[inline] #[must_use] - #[stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] - #[rustc_const_stable(feature = "inherent_str_constructors", since = "CURRENT_RUSTC_VERSION")] + #[stable(feature = "inherent_str_constructors", since = "1.87.0")] + #[rustc_const_stable(feature = "inherent_str_constructors", since = "1.87.0")] #[rustc_diagnostic_item = "str_inherent_from_utf8_unchecked_mut"] pub const unsafe fn from_utf8_unchecked_mut(v: &mut [u8]) -> &mut str { // SAFETY: converts::from_utf8_unchecked_mut has the same safety requirements as this function. diff --git a/library/core/src/tuple.rs b/library/core/src/tuple.rs index d754bb90343..02eb805ece1 100644 --- a/library/core/src/tuple.rs +++ b/library/core/src/tuple.rs @@ -2,7 +2,7 @@ use crate::cmp::Ordering::{self, *}; use crate::marker::{ConstParamTy_, StructuralPartialEq, UnsizedConstParamTy}; -use crate::ops::ControlFlow::{Break, Continue}; +use crate::ops::ControlFlow::{self, Break, Continue}; // Recursive macro for implementing n-ary tuple functions and operations // @@ -95,6 +95,22 @@ macro_rules! tuple_impls { fn gt(&self, other: &($($T,)+)) -> bool { lexical_ord!(gt, __chaining_gt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } + #[inline] + fn __chaining_lt(&self, other: &($($T,)+)) -> ControlFlow<bool> { + lexical_chain!(__chaining_lt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + } + #[inline] + fn __chaining_le(&self, other: &($($T,)+)) -> ControlFlow<bool> { + lexical_chain!(__chaining_le, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + } + #[inline] + fn __chaining_gt(&self, other: &($($T,)+)) -> ControlFlow<bool> { + lexical_chain!(__chaining_gt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + } + #[inline] + fn __chaining_ge(&self, other: &($($T,)+)) -> ControlFlow<bool> { + lexical_chain!(__chaining_ge, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + } } } @@ -187,6 +203,17 @@ macro_rules! lexical_ord { }; } +// Same parameter interleaving as `lexical_ord` above +macro_rules! lexical_chain { + ($chain_rel: ident, $a:expr, $b:expr $(,$rest_a:expr, $rest_b:expr)*) => {{ + PartialOrd::$chain_rel(&$a, &$b)?; + lexical_chain!($chain_rel $(,$rest_a, $rest_b)*) + }}; + ($chain_rel: ident) => { + Continue(()) + }; +} + macro_rules! lexical_partial_cmp { ($a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => { match ($a).partial_cmp(&$b) { |