From 596aabe5c7dd00a037a8aa5fd41b929010ebb7ae Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 23 Sep 2023 00:19:36 -0400 Subject: Add num, ptr, and cmp modules --- crates/core_simd/examples/dot_product.rs | 2 +- crates/core_simd/examples/matrix_inversion.rs | 6 +- crates/core_simd/examples/nbody.rs | 2 +- crates/core_simd/examples/spectral_norm.rs | 2 +- crates/core_simd/src/core_simd_docs.md | 2 +- crates/core_simd/src/elements.rs | 15 - crates/core_simd/src/elements/const_ptr.rs | 163 ---------- crates/core_simd/src/elements/float.rs | 425 -------------------------- crates/core_simd/src/elements/int.rs | 371 ---------------------- crates/core_simd/src/elements/mut_ptr.rs | 158 ---------- crates/core_simd/src/elements/uint.rs | 221 -------------- crates/core_simd/src/eq.rs | 109 ------- crates/core_simd/src/masks.rs | 4 +- crates/core_simd/src/mod.rs | 12 +- crates/core_simd/src/ops.rs | 2 +- crates/core_simd/src/ord.rs | 317 ------------------- crates/core_simd/src/simd/cmp.rs | 7 + crates/core_simd/src/simd/cmp/eq.rs | 111 +++++++ crates/core_simd/src/simd/cmp/ord.rs | 320 +++++++++++++++++++ crates/core_simd/src/simd/num.rs | 13 + crates/core_simd/src/simd/num/float.rs | 425 ++++++++++++++++++++++++++ crates/core_simd/src/simd/num/int.rs | 371 ++++++++++++++++++++++ crates/core_simd/src/simd/num/uint.rs | 221 ++++++++++++++ crates/core_simd/src/simd/prelude.rs | 6 +- crates/core_simd/src/simd/ptr.rs | 11 + crates/core_simd/src/simd/ptr/const_ptr.rs | 165 ++++++++++ crates/core_simd/src/simd/ptr/mut_ptr.rs | 160 ++++++++++ crates/core_simd/src/swizzle_dyn.rs | 4 +- crates/core_simd/src/to_bytes.rs | 5 +- crates/core_simd/src/vector.rs | 18 +- crates/core_simd/tests/cast.rs | 2 +- crates/core_simd/tests/ops_macros.rs | 12 +- crates/core_simd/tests/pointers.rs | 5 +- crates/core_simd/tests/round.rs | 2 +- crates/std_float/src/lib.rs | 2 +- 35 files changed, 1855 insertions(+), 1816 deletions(-) delete mode 100644 crates/core_simd/src/elements.rs delete mode 100644 crates/core_simd/src/elements/const_ptr.rs delete mode 100644 crates/core_simd/src/elements/float.rs delete mode 100644 crates/core_simd/src/elements/int.rs delete mode 100644 crates/core_simd/src/elements/mut_ptr.rs delete mode 100644 crates/core_simd/src/elements/uint.rs delete mode 100644 crates/core_simd/src/eq.rs delete mode 100644 crates/core_simd/src/ord.rs create mode 100644 crates/core_simd/src/simd/cmp.rs create mode 100644 crates/core_simd/src/simd/cmp/eq.rs create mode 100644 crates/core_simd/src/simd/cmp/ord.rs create mode 100644 crates/core_simd/src/simd/num.rs create mode 100644 crates/core_simd/src/simd/num/float.rs create mode 100644 crates/core_simd/src/simd/num/int.rs create mode 100644 crates/core_simd/src/simd/num/uint.rs create mode 100644 crates/core_simd/src/simd/ptr.rs create mode 100644 crates/core_simd/src/simd/ptr/const_ptr.rs create mode 100644 crates/core_simd/src/simd/ptr/mut_ptr.rs diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 391f08f55a0..e5815888bb7 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -6,7 +6,7 @@ #![feature(slice_as_chunks)] // Add these imports to use the stdsimd library #![feature(portable_simd)] -use core_simd::simd::*; +use core_simd::simd::prelude::*; // This is your barebones dot product implementation: // Take 2 vectors, multiply them element wise and *then* diff --git a/crates/core_simd/examples/matrix_inversion.rs b/crates/core_simd/examples/matrix_inversion.rs index 39f530f68f5..5176623c160 100644 --- a/crates/core_simd/examples/matrix_inversion.rs +++ b/crates/core_simd/examples/matrix_inversion.rs @@ -2,8 +2,10 @@ // Code ported from the `packed_simd` crate // Run this code with `cargo test --example matrix_inversion` #![feature(array_chunks, portable_simd)] -use core_simd::simd::*; -use Which::*; +use core_simd::simd::{ + prelude::*, + Which::{self, *}, +}; // Gotta define our own 4x4 matrix since Rust doesn't ship multidim arrays yet :^) #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] diff --git a/crates/core_simd/examples/nbody.rs b/crates/core_simd/examples/nbody.rs index df38a00967f..154e24c460e 100644 --- a/crates/core_simd/examples/nbody.rs +++ b/crates/core_simd/examples/nbody.rs @@ -5,7 +5,7 @@ extern crate std_float; /// Taken from the `packed_simd` crate /// Run this benchmark with `cargo test --example nbody` mod nbody { - use core_simd::simd::*; + use core_simd::simd::prelude::*; #[allow(unused)] // False positive? use std_float::StdFloat; diff --git a/crates/core_simd/examples/spectral_norm.rs b/crates/core_simd/examples/spectral_norm.rs index d576bd0ccee..bc7934c2522 100644 --- a/crates/core_simd/examples/spectral_norm.rs +++ b/crates/core_simd/examples/spectral_norm.rs @@ -1,6 +1,6 @@ #![feature(portable_simd)] -use core_simd::simd::*; +use core_simd::simd::prelude::*; fn a(i: usize, j: usize) -> f64 { ((i + j) * (i + j + 1) / 2 + i + 1) as f64 diff --git a/crates/core_simd/src/core_simd_docs.md b/crates/core_simd/src/core_simd_docs.md index 8acdeb04427..fa93155ff5e 100644 --- a/crates/core_simd/src/core_simd_docs.md +++ b/crates/core_simd/src/core_simd_docs.md @@ -30,7 +30,7 @@ Instead, they map to a reasonable implementation of the operation for the target Consistency between targets is not compromised to use faster or fewer instructions. In some cases, `std::arch` will provide a faster function that has slightly different behavior than the `std::simd` equivalent. -For example, [`_mm_min_ps`](`core::arch::x86_64::_mm_min_ps`)[^1] can be slightly faster than [`SimdFloat::simd_min`], but does not conform to the IEEE standard also used by [`f32::min`]. +For example, [`_mm_min_ps`](`core::arch::x86_64::_mm_min_ps`)[^1] can be slightly faster than [`SimdFloat::simd_min`](`num::SimdFloat::simd_min`), but does not conform to the IEEE standard also used by [`f32::min`]. When necessary, [`Simd`] can be converted to the types provided by `std::arch` to make use of target-specific functions. Many targets simply don't have SIMD, or don't support SIMD for a particular element type. diff --git a/crates/core_simd/src/elements.rs b/crates/core_simd/src/elements.rs deleted file mode 100644 index dc7f52a4d57..00000000000 --- a/crates/core_simd/src/elements.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod const_ptr; -mod float; -mod int; -mod mut_ptr; -mod uint; - -mod sealed { - pub trait Sealed {} -} - -pub use const_ptr::*; -pub use float::*; -pub use int::*; -pub use mut_ptr::*; -pub use uint::*; diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs deleted file mode 100644 index f215f9a61d0..00000000000 --- a/crates/core_simd/src/elements/const_ptr.rs +++ /dev/null @@ -1,163 +0,0 @@ -use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount}; - -/// Operations on SIMD vectors of constant pointers. -pub trait SimdConstPtr: Copy + Sealed { - /// Vector of `usize` with the same number of lanes. - type Usize; - - /// Vector of `isize` with the same number of lanes. - type Isize; - - /// Vector of const pointers with the same number of lanes. - type CastPtr; - - /// Vector of mutable pointers to the same type. - type MutPtr; - - /// Mask type used for manipulating this SIMD vector type. - type Mask; - - /// Returns `true` for each lane that is null. - fn is_null(self) -> Self::Mask; - - /// Casts to a pointer of another type. - /// - /// Equivalent to calling [`pointer::cast`] on each lane. - fn cast(self) -> Self::CastPtr; - - /// Changes constness without changing the type. - /// - /// Equivalent to calling [`pointer::cast_mut`] on each lane. - fn cast_mut(self) -> Self::MutPtr; - - /// Gets the "address" portion of the pointer. - /// - /// This method discards pointer semantic metadata, so the result cannot be - /// directly cast into a valid pointer. - /// - /// This method semantically discards *provenance* and - /// *address-space* information. To properly restore that information, use [`Self::with_addr`]. - /// - /// Equivalent to calling [`pointer::addr`] on each lane. - fn addr(self) -> Self::Usize; - - /// Creates a new pointer with the given address. - /// - /// This performs the same operation as a cast, but copies the *address-space* and - /// *provenance* of `self` to the new pointer. - /// - /// Equivalent to calling [`pointer::with_addr`] on each lane. - fn with_addr(self, addr: Self::Usize) -> Self; - - /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use - /// in [`Self::from_exposed_addr`]. - fn expose_addr(self) -> Self::Usize; - - /// Convert an address back to a pointer, picking up a previously "exposed" provenance. - /// - /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each lane. - fn from_exposed_addr(addr: Self::Usize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. - fn wrapping_offset(self, offset: Self::Isize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. - fn wrapping_add(self, count: Self::Usize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. - fn wrapping_sub(self, count: Self::Usize) -> Self; -} - -impl Sealed for Simd<*const T, LANES> where - LaneCount: SupportedLaneCount -{ -} - -impl SimdConstPtr for Simd<*const T, LANES> -where - LaneCount: SupportedLaneCount, -{ - type Usize = Simd; - type Isize = Simd; - type CastPtr = Simd<*const U, LANES>; - type MutPtr = Simd<*mut T, LANES>; - type Mask = Mask; - - #[inline] - fn is_null(self) -> Self::Mask { - Simd::splat(core::ptr::null()).simd_eq(self) - } - - #[inline] - fn cast(self) -> Self::CastPtr { - // SimdElement currently requires zero-sized metadata, so this should never fail. - // If this ever changes, `simd_cast_ptr` should produce a post-mono error. - use core::{mem::size_of, ptr::Pointee}; - assert_eq!(size_of::<::Metadata>(), 0); - assert_eq!(size_of::<::Metadata>(), 0); - - // Safety: pointers can be cast - unsafe { intrinsics::simd_cast_ptr(self) } - } - - #[inline] - fn cast_mut(self) -> Self::MutPtr { - // Safety: pointers can be cast - unsafe { intrinsics::simd_cast_ptr(self) } - } - - #[inline] - fn addr(self) -> Self::Usize { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the - // provenance). - unsafe { core::mem::transmute_copy(&self) } - } - - #[inline] - fn with_addr(self, addr: Self::Usize) -> Self { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // - // In the mean-time, this operation is defined to be "as if" it was - // a wrapping_offset, so we can emulate it as such. This should properly - // restore pointer provenance even under today's compiler. - self.cast::() - .wrapping_offset(addr.cast::() - self.addr().cast::()) - .cast() - } - - #[inline] - fn expose_addr(self) -> Self::Usize { - // Safety: `self` is a pointer vector - unsafe { intrinsics::simd_expose_addr(self) } - } - - #[inline] - fn from_exposed_addr(addr: Self::Usize) -> Self { - // Safety: `self` is a pointer vector - unsafe { intrinsics::simd_from_exposed_addr(addr) } - } - - #[inline] - fn wrapping_offset(self, count: Self::Isize) -> Self { - // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets - unsafe { intrinsics::simd_arith_offset(self, count) } - } - - #[inline] - fn wrapping_add(self, count: Self::Usize) -> Self { - self.wrapping_offset(count.cast()) - } - - #[inline] - fn wrapping_sub(self, count: Self::Usize) -> Self { - self.wrapping_offset(-count.cast::()) - } -} diff --git a/crates/core_simd/src/elements/float.rs b/crates/core_simd/src/elements/float.rs deleted file mode 100644 index d700011ff9c..00000000000 --- a/crates/core_simd/src/elements/float.rs +++ /dev/null @@ -1,425 +0,0 @@ -use super::sealed::Sealed; -use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialEq, SimdPartialOrd, - SupportedLaneCount, -}; - -/// Operations on SIMD vectors of floats. -pub trait SimdFloat: Copy + Sealed { - /// Mask type used for manipulating this SIMD vector type. - type Mask; - - /// Scalar type contained by this SIMD vector type. - type Scalar; - - /// Bit representation of this SIMD vector type. - type Bits; - - /// A SIMD vector with a different element type. - type Cast; - - /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. - /// - /// This follows the semantics of Rust's `as` conversion for floats (truncating or saturating - /// at the limits) for each element. - /// - /// # Example - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{SimdFloat, SimdInt, Simd}; - /// let floats: Simd = Simd::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]); - /// let ints = floats.cast::(); - /// assert_eq!(ints, Simd::from_array([1, -4, i32::MAX, 0])); - /// - /// // Formally equivalent, but `Simd::cast` can optimize better. - /// assert_eq!(ints, Simd::from_array(floats.to_array().map(|x| x as i32))); - /// - /// // The float conversion does not round-trip. - /// let floats_again = ints.cast(); - /// assert_ne!(floats, floats_again); - /// assert_eq!(floats_again, Simd::from_array([1.0, -4.0, 2147483647.0, 0.0])); - /// ``` - #[must_use] - fn cast(self) -> Self::Cast; - - /// Rounds toward zero and converts to the same-width integer type, assuming that - /// the value is finite and fits in that type. - /// - /// # Safety - /// The value must: - /// - /// * Not be NaN - /// * Not be infinite - /// * Be representable in the return type, after truncating off its fractional part - /// - /// If these requirements are infeasible or costly, consider using the safe function [cast], - /// which saturates on conversion. - /// - /// [cast]: Simd::cast - unsafe fn to_int_unchecked(self) -> Self::Cast - where - Self::Scalar: core::convert::FloatToInt; - - /// Raw transmutation to an unsigned integer vector type with the - /// same size and number of lanes. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn to_bits(self) -> Self::Bits; - - /// Raw transmutation from an unsigned integer vector type with the - /// same size and number of lanes. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn from_bits(bits: Self::Bits) -> Self; - - /// Produces a vector where every lane has the absolute value of the - /// equivalently-indexed lane in `self`. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn abs(self) -> Self; - - /// Takes the reciprocal (inverse) of each lane, `1/x`. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn recip(self) -> Self; - - /// Converts each lane from radians to degrees. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn to_degrees(self) -> Self; - - /// Converts each lane from degrees to radians. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn to_radians(self) -> Self; - - /// Returns true for each lane if it has a positive sign, including - /// `+0.0`, `NaN`s with positive sign bit and positive infinity. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_sign_positive(self) -> Self::Mask; - - /// Returns true for each lane if it has a negative sign, including - /// `-0.0`, `NaN`s with negative sign bit and negative infinity. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_sign_negative(self) -> Self::Mask; - - /// Returns true for each lane if its value is `NaN`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_nan(self) -> Self::Mask; - - /// Returns true for each lane if its value is positive infinity or negative infinity. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_infinite(self) -> Self::Mask; - - /// Returns true for each lane if its value is neither infinite nor `NaN`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_finite(self) -> Self::Mask; - - /// Returns true for each lane if its value is subnormal. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_subnormal(self) -> Self::Mask; - - /// Returns true for each lane if its value is neither zero, infinite, - /// subnormal, nor `NaN`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_normal(self) -> Self::Mask; - - /// Replaces each lane with a number that represents its sign. - /// - /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` - /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` - /// * `NAN` if the number is `NAN` - #[must_use = "method returns a new vector and does not mutate the original value"] - fn signum(self) -> Self; - - /// Returns each lane with the magnitude of `self` and the sign of `sign`. - /// - /// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn copysign(self, sign: Self) -> Self; - - /// Returns the minimum of each lane. - /// - /// If one of the values is `NAN`, then the other value is returned. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_min(self, other: Self) -> Self; - - /// Returns the maximum of each lane. - /// - /// If one of the values is `NAN`, then the other value is returned. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_max(self, other: Self) -> Self; - - /// Restrict each lane to a certain interval unless it is NaN. - /// - /// For each lane in `self`, returns the corresponding lane in `max` if the lane is - /// greater than `max`, and the corresponding lane in `min` if the lane is less - /// than `min`. Otherwise returns the lane in `self`. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_clamp(self, min: Self, max: Self) -> Self; - - /// Returns the sum of the lanes of the vector. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; - /// let v = f32x2::from_array([1., 2.]); - /// assert_eq!(v.reduce_sum(), 3.); - /// ``` - fn reduce_sum(self) -> Self::Scalar; - - /// Reducing multiply. Returns the product of the lanes of the vector. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; - /// let v = f32x2::from_array([3., 4.]); - /// assert_eq!(v.reduce_product(), 12.); - /// ``` - fn reduce_product(self) -> Self::Scalar; - - /// Returns the maximum lane in the vector. - /// - /// Returns values based on equality, so a vector containing both `0.` and `-0.` may - /// return either. - /// - /// This function will not return `NaN` unless all lanes are `NaN`. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; - /// let v = f32x2::from_array([1., 2.]); - /// assert_eq!(v.reduce_max(), 2.); - /// - /// // NaN values are skipped... - /// let v = f32x2::from_array([1., f32::NAN]); - /// assert_eq!(v.reduce_max(), 1.); - /// - /// // ...unless all values are NaN - /// let v = f32x2::from_array([f32::NAN, f32::NAN]); - /// assert!(v.reduce_max().is_nan()); - /// ``` - fn reduce_max(self) -> Self::Scalar; - - /// Returns the minimum lane in the vector. - /// - /// Returns values based on equality, so a vector containing both `0.` and `-0.` may - /// return either. - /// - /// This function will not return `NaN` unless all lanes are `NaN`. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{f32x2, SimdFloat}; - /// let v = f32x2::from_array([3., 7.]); - /// assert_eq!(v.reduce_min(), 3.); - /// - /// // NaN values are skipped... - /// let v = f32x2::from_array([1., f32::NAN]); - /// assert_eq!(v.reduce_min(), 1.); - /// - /// // ...unless all values are NaN - /// let v = f32x2::from_array([f32::NAN, f32::NAN]); - /// assert!(v.reduce_min().is_nan()); - /// ``` - fn reduce_min(self) -> Self::Scalar; -} - -macro_rules! impl_trait { - { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { - $( - impl Sealed for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - } - - impl SimdFloat for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>; - type Scalar = $ty; - type Bits = Simd<$bits_ty, LANES>; - type Cast = Simd; - - #[inline] - fn cast(self) -> Self::Cast - { - // Safety: supported types are guaranteed by SimdCast - unsafe { intrinsics::simd_as(self) } - } - - #[inline] - #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - unsafe fn to_int_unchecked(self) -> Self::Cast - where - Self::Scalar: core::convert::FloatToInt, - { - // Safety: supported types are guaranteed by SimdCast, the caller is responsible for the extra invariants - unsafe { intrinsics::simd_cast(self) } - } - - #[inline] - fn to_bits(self) -> Simd<$bits_ty, LANES> { - assert_eq!(core::mem::size_of::(), core::mem::size_of::()); - // Safety: transmuting between vector types is safe - unsafe { core::mem::transmute_copy(&self) } - } - - #[inline] - fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { - assert_eq!(core::mem::size_of::(), core::mem::size_of::()); - // Safety: transmuting between vector types is safe - unsafe { core::mem::transmute_copy(&bits) } - } - - #[inline] - fn abs(self) -> Self { - // Safety: `self` is a float vector - unsafe { intrinsics::simd_fabs(self) } - } - - #[inline] - fn recip(self) -> Self { - Self::splat(1.0) / self - } - - #[inline] - fn to_degrees(self) -> Self { - // to_degrees uses a special constant for better precision, so extract that constant - self * Self::splat(Self::Scalar::to_degrees(1.)) - } - - #[inline] - fn to_radians(self) -> Self { - self * Self::splat(Self::Scalar::to_radians(1.)) - } - - #[inline] - fn is_sign_positive(self) -> Self::Mask { - !self.is_sign_negative() - } - - #[inline] - fn is_sign_negative(self) -> Self::Mask { - let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1); - sign_bits.simd_gt(Simd::splat(0)) - } - - #[inline] - fn is_nan(self) -> Self::Mask { - self.simd_ne(self) - } - - #[inline] - fn is_infinite(self) -> Self::Mask { - self.abs().simd_eq(Self::splat(Self::Scalar::INFINITY)) - } - - #[inline] - fn is_finite(self) -> Self::Mask { - self.abs().simd_lt(Self::splat(Self::Scalar::INFINITY)) - } - - #[inline] - fn is_subnormal(self) -> Self::Mask { - // On some architectures (e.g. armv7 and some ppc) subnormals are flushed to zero, - // so this comparison must be done with integers. - let not_zero = self.abs().to_bits().simd_ne(Self::splat(0.0).to_bits()); - not_zero & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) - } - - #[inline] - #[must_use = "method returns a new mask and does not mutate the original value"] - fn is_normal(self) -> Self::Mask { - !(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) - } - - #[inline] - fn signum(self) -> Self { - self.is_nan().select(Self::splat(Self::Scalar::NAN), Self::splat(1.0).copysign(self)) - } - - #[inline] - fn copysign(self, sign: Self) -> Self { - let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits(); - let magnitude = self.to_bits() & !Self::splat(-0.).to_bits(); - Self::from_bits(sign_bit | magnitude) - } - - #[inline] - fn simd_min(self, other: Self) -> Self { - // Safety: `self` and `other` are float vectors - unsafe { intrinsics::simd_fmin(self, other) } - } - - #[inline] - fn simd_max(self, other: Self) -> Self { - // Safety: `self` and `other` are floating point vectors - unsafe { intrinsics::simd_fmax(self, other) } - } - - #[inline] - fn simd_clamp(self, min: Self, max: Self) -> Self { - assert!( - min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - let mut x = self; - x = x.simd_lt(min).select(min, x); - x = x.simd_gt(max).select(max, x); - x - } - - #[inline] - fn reduce_sum(self) -> Self::Scalar { - // LLVM sum is inaccurate on i586 - if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { - self.as_array().iter().sum() - } else { - // Safety: `self` is a float vector - unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) } - } - } - - #[inline] - fn reduce_product(self) -> Self::Scalar { - // LLVM product is inaccurate on i586 - if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { - self.as_array().iter().product() - } else { - // Safety: `self` is a float vector - unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) } - } - } - - #[inline] - fn reduce_max(self) -> Self::Scalar { - // Safety: `self` is a float vector - unsafe { intrinsics::simd_reduce_max(self) } - } - - #[inline] - fn reduce_min(self) -> Self::Scalar { - // Safety: `self` is a float vector - unsafe { intrinsics::simd_reduce_min(self) } - } - } - )* - } -} - -impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } diff --git a/crates/core_simd/src/elements/int.rs b/crates/core_simd/src/elements/int.rs deleted file mode 100644 index c341c59545c..00000000000 --- a/crates/core_simd/src/elements/int.rs +++ /dev/null @@ -1,371 +0,0 @@ -use super::sealed::Sealed; -use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SimdPartialOrd, SimdUint, - SupportedLaneCount, -}; - -/// Operations on SIMD vectors of signed integers. -pub trait SimdInt: Copy + Sealed { - /// Mask type used for manipulating this SIMD vector type. - type Mask; - - /// Scalar type contained by this SIMD vector type. - type Scalar; - - /// A SIMD vector of unsigned integers with the same element size. - type Unsigned; - - /// A SIMD vector with a different element type. - type Cast; - - /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. - /// - /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to - /// other integer types, and saturating to float types). - #[must_use] - fn cast(self) -> Self::Cast; - - /// Lanewise saturating add. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; - /// use core::i32::{MIN, MAX}; - /// let x = Simd::from_array([MIN, 0, 1, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x + max; - /// let sat = x.saturating_add(max); - /// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2])); - /// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX])); - /// ``` - fn saturating_add(self, second: Self) -> Self; - - /// Lanewise saturating subtract. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; - /// use core::i32::{MIN, MAX}; - /// let x = Simd::from_array([MIN, -2, -1, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x - max; - /// let sat = x.saturating_sub(max); - /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); - /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); - fn saturating_sub(self, second: Self) -> Self; - - /// Lanewise absolute value, implemented in Rust. - /// Every lane becomes its absolute value. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; - /// use core::i32::{MIN, MAX}; - /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); - /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); - /// ``` - fn abs(self) -> Self; - - /// Lanewise saturating absolute value, implemented in Rust. - /// As abs(), except the MIN value becomes MAX instead of itself. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; - /// use core::i32::{MIN, MAX}; - /// let xs = Simd::from_array([MIN, -2, 0, 3]); - /// let unsat = xs.abs(); - /// let sat = xs.saturating_abs(); - /// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3])); - /// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3])); - /// ``` - fn saturating_abs(self) -> Self; - - /// Lanewise saturating negation, implemented in Rust. - /// As neg(), except the MIN value becomes MAX instead of itself. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdInt}; - /// use core::i32::{MIN, MAX}; - /// let x = Simd::from_array([MIN, -2, 3, MAX]); - /// let unsat = -x; - /// let sat = x.saturating_neg(); - /// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1])); - /// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1])); - /// ``` - fn saturating_neg(self) -> Self; - - /// Returns true for each positive lane and false if it is zero or negative. - fn is_positive(self) -> Self::Mask; - - /// Returns true for each negative lane and false if it is zero or positive. - fn is_negative(self) -> Self::Mask; - - /// Returns numbers representing the sign of each lane. - /// * `0` if the number is zero - /// * `1` if the number is positive - /// * `-1` if the number is negative - fn signum(self) -> Self; - - /// Returns the sum of the lanes of the vector, with wrapping addition. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; - /// let v = i32x4::from_array([1, 2, 3, 4]); - /// assert_eq!(v.reduce_sum(), 10); - /// - /// // SIMD integer addition is always wrapping - /// let v = i32x4::from_array([i32::MAX, 1, 0, 0]); - /// assert_eq!(v.reduce_sum(), i32::MIN); - /// ``` - fn reduce_sum(self) -> Self::Scalar; - - /// Returns the product of the lanes of the vector, with wrapping multiplication. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; - /// let v = i32x4::from_array([1, 2, 3, 4]); - /// assert_eq!(v.reduce_product(), 24); - /// - /// // SIMD integer multiplication is always wrapping - /// let v = i32x4::from_array([i32::MAX, 2, 1, 1]); - /// assert!(v.reduce_product() < i32::MAX); - /// ``` - fn reduce_product(self) -> Self::Scalar; - - /// Returns the maximum lane in the vector. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; - /// let v = i32x4::from_array([1, 2, 3, 4]); - /// assert_eq!(v.reduce_max(), 4); - /// ``` - fn reduce_max(self) -> Self::Scalar; - - /// Returns the minimum lane in the vector. - /// - /// # Examples - /// - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{i32x4, SimdInt}; - /// let v = i32x4::from_array([1, 2, 3, 4]); - /// assert_eq!(v.reduce_min(), 1); - /// ``` - fn reduce_min(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "and" across the lanes of the vector. - fn reduce_and(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "or" across the lanes of the vector. - fn reduce_or(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "xor" across the lanes of the vector. - fn reduce_xor(self) -> Self::Scalar; - - /// Reverses the byte order of each element. - fn swap_bytes(self) -> Self; - - /// Reverses the order of bits in each elemnent. - /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. - fn reverse_bits(self) -> Self; - - /// Returns the number of leading zeros in the binary representation of each element. - fn leading_zeros(self) -> Self::Unsigned; - - /// Returns the number of trailing zeros in the binary representation of each element. - fn trailing_zeros(self) -> Self::Unsigned; - - /// Returns the number of leading ones in the binary representation of each element. - fn leading_ones(self) -> Self::Unsigned; - - /// Returns the number of trailing ones in the binary representation of each element. - fn trailing_ones(self) -> Self::Unsigned; -} - -macro_rules! impl_trait { - { $($ty:ident ($unsigned:ident)),* } => { - $( - impl Sealed for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - } - - impl SimdInt for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - type Mask = Mask<<$ty as SimdElement>::Mask, LANES>; - type Scalar = $ty; - type Unsigned = Simd<$unsigned, LANES>; - type Cast = Simd; - - #[inline] - fn cast(self) -> Self::Cast { - // Safety: supported types are guaranteed by SimdCast - unsafe { intrinsics::simd_as(self) } - } - - #[inline] - fn saturating_add(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { intrinsics::simd_saturating_add(self, second) } - } - - #[inline] - fn saturating_sub(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { intrinsics::simd_saturating_sub(self, second) } - } - - #[inline] - fn abs(self) -> Self { - const SHR: $ty = <$ty>::BITS as $ty - 1; - let m = self >> Simd::splat(SHR); - (self^m) - m - } - - #[inline] - fn saturating_abs(self) -> Self { - // arith shift for -1 or 0 mask based on sign bit, giving 2s complement - const SHR: $ty = <$ty>::BITS as $ty - 1; - let m = self >> Simd::splat(SHR); - (self^m).saturating_sub(m) - } - - #[inline] - fn saturating_neg(self) -> Self { - Self::splat(0).saturating_sub(self) - } - - #[inline] - fn is_positive(self) -> Self::Mask { - self.simd_gt(Self::splat(0)) - } - - #[inline] - fn is_negative(self) -> Self::Mask { - self.simd_lt(Self::splat(0)) - } - - #[inline] - fn signum(self) -> Self { - self.is_positive().select( - Self::splat(1), - self.is_negative().select(Self::splat(-1), Self::splat(0)) - ) - } - - #[inline] - fn reduce_sum(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } - } - - #[inline] - fn reduce_product(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } - } - - #[inline] - fn reduce_max(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_max(self) } - } - - #[inline] - fn reduce_min(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_min(self) } - } - - #[inline] - fn reduce_and(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_and(self) } - } - - #[inline] - fn reduce_or(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_or(self) } - } - - #[inline] - fn reduce_xor(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_xor(self) } - } - - #[inline] - fn swap_bytes(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_bswap(self) } - } - - #[inline] - fn reverse_bits(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_bitreverse(self) } - } - - #[inline] - fn leading_zeros(self) -> Self::Unsigned { - self.cast::<$unsigned>().leading_zeros() - } - - #[inline] - fn trailing_zeros(self) -> Self::Unsigned { - self.cast::<$unsigned>().trailing_zeros() - } - - #[inline] - fn leading_ones(self) -> Self::Unsigned { - self.cast::<$unsigned>().leading_ones() - } - - #[inline] - fn trailing_ones(self) -> Self::Unsigned { - self.cast::<$unsigned>().trailing_ones() - } - } - )* - } -} - -impl_trait! { i8 (u8), i16 (u16), i32 (u32), i64 (u64), isize (usize) } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs deleted file mode 100644 index 4bdc6a14ce4..00000000000 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ /dev/null @@ -1,158 +0,0 @@ -use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SimdUint, SupportedLaneCount}; - -/// Operations on SIMD vectors of mutable pointers. -pub trait SimdMutPtr: Copy + Sealed { - /// Vector of `usize` with the same number of lanes. - type Usize; - - /// Vector of `isize` with the same number of lanes. - type Isize; - - /// Vector of const pointers with the same number of lanes. - type CastPtr; - - /// Vector of constant pointers to the same type. - type ConstPtr; - - /// Mask type used for manipulating this SIMD vector type. - type Mask; - - /// Returns `true` for each lane that is null. - fn is_null(self) -> Self::Mask; - - /// Casts to a pointer of another type. - /// - /// Equivalent to calling [`pointer::cast`] on each lane. - fn cast(self) -> Self::CastPtr; - - /// Changes constness without changing the type. - /// - /// Equivalent to calling [`pointer::cast_const`] on each lane. - fn cast_const(self) -> Self::ConstPtr; - - /// Gets the "address" portion of the pointer. - /// - /// This method discards pointer semantic metadata, so the result cannot be - /// directly cast into a valid pointer. - /// - /// Equivalent to calling [`pointer::addr`] on each lane. - fn addr(self) -> Self::Usize; - - /// Creates a new pointer with the given address. - /// - /// This performs the same operation as a cast, but copies the *address-space* and - /// *provenance* of `self` to the new pointer. - /// - /// Equivalent to calling [`pointer::with_addr`] on each lane. - fn with_addr(self, addr: Self::Usize) -> Self; - - /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use - /// in [`Self::from_exposed_addr`]. - fn expose_addr(self) -> Self::Usize; - - /// Convert an address back to a pointer, picking up a previously "exposed" provenance. - /// - /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each lane. - fn from_exposed_addr(addr: Self::Usize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. - fn wrapping_offset(self, offset: Self::Isize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. - fn wrapping_add(self, count: Self::Usize) -> Self; - - /// Calculates the offset from a pointer using wrapping arithmetic. - /// - /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. - fn wrapping_sub(self, count: Self::Usize) -> Self; -} - -impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount -{} - -impl SimdMutPtr for Simd<*mut T, LANES> -where - LaneCount: SupportedLaneCount, -{ - type Usize = Simd; - type Isize = Simd; - type CastPtr = Simd<*mut U, LANES>; - type ConstPtr = Simd<*const T, LANES>; - type Mask = Mask; - - #[inline] - fn is_null(self) -> Self::Mask { - Simd::splat(core::ptr::null_mut()).simd_eq(self) - } - - #[inline] - fn cast(self) -> Self::CastPtr { - // SimdElement currently requires zero-sized metadata, so this should never fail. - // If this ever changes, `simd_cast_ptr` should produce a post-mono error. - use core::{mem::size_of, ptr::Pointee}; - assert_eq!(size_of::<::Metadata>(), 0); - assert_eq!(size_of::<::Metadata>(), 0); - - // Safety: pointers can be cast - unsafe { intrinsics::simd_cast_ptr(self) } - } - - #[inline] - fn cast_const(self) -> Self::ConstPtr { - // Safety: pointers can be cast - unsafe { intrinsics::simd_cast_ptr(self) } - } - - #[inline] - fn addr(self) -> Self::Usize { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the - // provenance). - unsafe { core::mem::transmute_copy(&self) } - } - - #[inline] - fn with_addr(self, addr: Self::Usize) -> Self { - // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. - // - // In the mean-time, this operation is defined to be "as if" it was - // a wrapping_offset, so we can emulate it as such. This should properly - // restore pointer provenance even under today's compiler. - self.cast::() - .wrapping_offset(addr.cast::() - self.addr().cast::()) - .cast() - } - - #[inline] - fn expose_addr(self) -> Self::Usize { - // Safety: `self` is a pointer vector - unsafe { intrinsics::simd_expose_addr(self) } - } - - #[inline] - fn from_exposed_addr(addr: Self::Usize) -> Self { - // Safety: `self` is a pointer vector - unsafe { intrinsics::simd_from_exposed_addr(addr) } - } - - #[inline] - fn wrapping_offset(self, count: Self::Isize) -> Self { - // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets - unsafe { intrinsics::simd_arith_offset(self, count) } - } - - #[inline] - fn wrapping_add(self, count: Self::Usize) -> Self { - self.wrapping_offset(count.cast()) - } - - #[inline] - fn wrapping_sub(self, count: Self::Usize) -> Self { - self.wrapping_offset(-count.cast::()) - } -} diff --git a/crates/core_simd/src/elements/uint.rs b/crates/core_simd/src/elements/uint.rs deleted file mode 100644 index c33059f7d4e..00000000000 --- a/crates/core_simd/src/elements/uint.rs +++ /dev/null @@ -1,221 +0,0 @@ -use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; - -/// Operations on SIMD vectors of unsigned integers. -pub trait SimdUint: Copy + Sealed { - /// Scalar type contained by this SIMD vector type. - type Scalar; - - /// A SIMD vector with a different element type. - type Cast; - - /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. - /// - /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to - /// other integer types, and saturating to float types). - #[must_use] - fn cast(self) -> Self::Cast; - - /// Wrapping negation. - /// - /// Like [`u32::wrapping_neg`], all applications of this function will wrap, with the exception - /// of `-0`. - fn wrapping_neg(self) -> Self; - - /// Lanewise saturating add. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdUint}; - /// use core::u32::MAX; - /// let x = Simd::from_array([2, 1, 0, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x + max; - /// let sat = x.saturating_add(max); - /// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1])); - /// assert_eq!(sat, max); - /// ``` - fn saturating_add(self, second: Self) -> Self; - - /// Lanewise saturating subtract. - /// - /// # Examples - /// ``` - /// # #![feature(portable_simd)] - /// # #[cfg(feature = "as_crate")] use core_simd::simd; - /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdUint}; - /// use core::u32::MAX; - /// let x = Simd::from_array([2, 1, 0, MAX]); - /// let max = Simd::splat(MAX); - /// let unsat = x - max; - /// let sat = x.saturating_sub(max); - /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); - /// assert_eq!(sat, Simd::splat(0)); - fn saturating_sub(self, second: Self) -> Self; - - /// Returns the sum of the lanes of the vector, with wrapping addition. - fn reduce_sum(self) -> Self::Scalar; - - /// Returns the product of the lanes of the vector, with wrapping multiplication. - fn reduce_product(self) -> Self::Scalar; - - /// Returns the maximum lane in the vector. - fn reduce_max(self) -> Self::Scalar; - - /// Returns the minimum lane in the vector. - fn reduce_min(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "and" across the lanes of the vector. - fn reduce_and(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "or" across the lanes of the vector. - fn reduce_or(self) -> Self::Scalar; - - /// Returns the cumulative bitwise "xor" across the lanes of the vector. - fn reduce_xor(self) -> Self::Scalar; - - /// Reverses the byte order of each element. - fn swap_bytes(self) -> Self; - - /// Reverses the order of bits in each elemnent. - /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. - fn reverse_bits(self) -> Self; - - /// Returns the number of leading zeros in the binary representation of each element. - fn leading_zeros(self) -> Self; - - /// Returns the number of trailing zeros in the binary representation of each element. - fn trailing_zeros(self) -> Self; - - /// Returns the number of leading ones in the binary representation of each element. - fn leading_ones(self) -> Self; - - /// Returns the number of trailing ones in the binary representation of each element. - fn trailing_ones(self) -> Self; -} - -macro_rules! impl_trait { - { $($ty:ident ($signed:ident)),* } => { - $( - impl Sealed for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - } - - impl SimdUint for Simd<$ty, LANES> - where - LaneCount: SupportedLaneCount, - { - type Scalar = $ty; - type Cast = Simd; - - #[inline] - fn cast(self) -> Self::Cast { - // Safety: supported types are guaranteed by SimdCast - unsafe { intrinsics::simd_as(self) } - } - - #[inline] - fn wrapping_neg(self) -> Self { - use crate::simd::SimdInt; - (-self.cast::<$signed>()).cast() - } - - #[inline] - fn saturating_add(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { intrinsics::simd_saturating_add(self, second) } - } - - #[inline] - fn saturating_sub(self, second: Self) -> Self { - // Safety: `self` is a vector - unsafe { intrinsics::simd_saturating_sub(self, second) } - } - - #[inline] - fn reduce_sum(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } - } - - #[inline] - fn reduce_product(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } - } - - #[inline] - fn reduce_max(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_max(self) } - } - - #[inline] - fn reduce_min(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_min(self) } - } - - #[inline] - fn reduce_and(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_and(self) } - } - - #[inline] - fn reduce_or(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_or(self) } - } - - #[inline] - fn reduce_xor(self) -> Self::Scalar { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_reduce_xor(self) } - } - - #[inline] - fn swap_bytes(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_bswap(self) } - } - - #[inline] - fn reverse_bits(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_bitreverse(self) } - } - - #[inline] - fn leading_zeros(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_ctlz(self) } - } - - #[inline] - fn trailing_zeros(self) -> Self { - // Safety: `self` is an integer vector - unsafe { intrinsics::simd_cttz(self) } - } - - #[inline] - fn leading_ones(self) -> Self { - (!self).leading_zeros() - } - - #[inline] - fn trailing_ones(self) -> Self { - (!self).trailing_zeros() - } - } - )* - } -} - -impl_trait! { u8 (i8), u16 (i16), u32 (i32), u64 (i64), usize (isize) } diff --git a/crates/core_simd/src/eq.rs b/crates/core_simd/src/eq.rs deleted file mode 100644 index 80763c07272..00000000000 --- a/crates/core_simd/src/eq.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdElement, SimdMutPtr, SupportedLaneCount, -}; - -/// Parallel `PartialEq`. -pub trait SimdPartialEq { - /// The mask type returned by each comparison. - type Mask; - - /// Test if each lane is equal to the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_eq(self, other: Self) -> Self::Mask; - - /// Test if each lane is equal to the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_ne(self, other: Self) -> Self::Mask; -} - -macro_rules! impl_number { - { $($number:ty),* } => { - $( - impl SimdPartialEq for Simd<$number, LANES> - where - LaneCount: SupportedLaneCount, - { - type Mask = Mask<<$number as SimdElement>::Mask, LANES>; - - #[inline] - fn simd_eq(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } - } - - #[inline] - fn simd_ne(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } - } - } - )* - } -} - -impl_number! { f32, f64, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } - -macro_rules! impl_mask { - { $($integer:ty),* } => { - $( - impl SimdPartialEq for Mask<$integer, LANES> - where - LaneCount: SupportedLaneCount, - { - type Mask = Self; - - #[inline] - fn simd_eq(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) } - } - - #[inline] - fn simd_ne(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) } - } - } - )* - } -} - -impl_mask! { i8, i16, i32, i64, isize } - -impl SimdPartialEq for Simd<*const T, LANES> -where - LaneCount: SupportedLaneCount, -{ - type Mask = Mask; - - #[inline] - fn simd_eq(self, other: Self) -> Self::Mask { - self.addr().simd_eq(other.addr()) - } - - #[inline] - fn simd_ne(self, other: Self) -> Self::Mask { - self.addr().simd_ne(other.addr()) - } -} - -impl SimdPartialEq for Simd<*mut T, LANES> -where - LaneCount: SupportedLaneCount, -{ - type Mask = Mask; - - #[inline] - fn simd_eq(self, other: Self) -> Self::Mask { - self.addr().simd_eq(other.addr()) - } - - #[inline] - fn simd_ne(self, other: Self) -> Self::Mask { - self.addr().simd_ne(other.addr()) - } -} diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 13ae5088fb9..0a04cf66757 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -15,7 +15,9 @@ mod mask_impl; mod to_bitmask; pub use to_bitmask::{ToBitMask, ToBitMaskArray}; -use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount, +}; use core::cmp::Ordering; use core::{fmt, mem}; diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index f489ae36de4..6fd458d24e7 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -5,14 +5,11 @@ pub(crate) mod intrinsics; mod alias; mod cast; -mod elements; -mod eq; mod fmt; mod iter; mod lane_count; mod masks; mod ops; -mod ord; mod select; mod swizzle_dyn; mod to_bytes; @@ -24,15 +21,18 @@ pub mod simd { pub mod prelude; + pub mod num; + + pub mod ptr; + + pub mod cmp; + pub(crate) use crate::core_simd::intrinsics; pub use crate::core_simd::alias::*; pub use crate::core_simd::cast::*; - pub use crate::core_simd::elements::*; - pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; pub use crate::core_simd::masks::*; - pub use crate::core_simd::ord::*; pub use crate::core_simd::swizzle::*; pub use crate::core_simd::swizzle_dyn::*; pub use crate::core_simd::to_bytes::ToBytes; diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index 63a96106283..d1b4a504884 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -1,4 +1,4 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{cmp::SimdPartialEq, LaneCount, Simd, SimdElement, SupportedLaneCount}; use core::ops::{Add, Mul}; use core::ops::{BitAnd, BitOr, BitXor}; use core::ops::{Div, Rem, Sub}; diff --git a/crates/core_simd/src/ord.rs b/crates/core_simd/src/ord.rs deleted file mode 100644 index b2455190e82..00000000000 --- a/crates/core_simd/src/ord.rs +++ /dev/null @@ -1,317 +0,0 @@ -use crate::simd::{ - intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdMutPtr, SimdPartialEq, SupportedLaneCount, -}; - -/// Parallel `PartialOrd`. -pub trait SimdPartialOrd: SimdPartialEq { - /// Test if each lane is less than the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_lt(self, other: Self) -> Self::Mask; - - /// Test if each lane is less than or equal to the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_le(self, other: Self) -> Self::Mask; - - /// Test if each lane is greater than the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_gt(self, other: Self) -> Self::Mask; - - /// Test if each lane is greater than or equal to the corresponding lane in `other`. - #[must_use = "method returns a new mask and does not mutate the original value"] - fn simd_ge(self, other: Self) -> Self::Mask; -} - -/// Parallel `Ord`. -pub trait SimdOrd: SimdPartialOrd { - /// Returns the lane-wise maximum with `other`. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_max(self, other: Self) -> Self; - - /// Returns the lane-wise minimum with `other`. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_min(self, other: Self) -> Self; - - /// Restrict each lane to a certain interval. - /// - /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is - /// less than `min`. Otherwise returns `self`. - /// - /// # Panics - /// - /// Panics if `min > max` on any lane. - #[must_use = "method returns a new vector and does not mutate the original value"] - fn simd_clamp(self, min: Self, max: Self) -> Self; -} - -macro_rules! impl_integer { - { $($integer:ty),* } => { - $( - impl SimdPartialOrd for Simd<$integer, LANES> - where - LaneCount: SupportedLaneCount, - { - #[inline] - fn simd_lt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } - } - - #[inline] - fn simd_le(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } - } - - #[inline] - fn simd_gt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } - } - - #[inline] - fn simd_ge(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } - } - } - - impl SimdOrd for Simd<$integer, LANES> - where - LaneCount: SupportedLaneCount, - { - #[inline] - fn simd_max(self, other: Self) -> Self { - self.simd_lt(other).select(other, self) - } - - #[inline] - fn simd_min(self, other: Self) -> Self { - self.simd_gt(other).select(other, self) - } - - #[inline] - #[track_caller] - fn simd_clamp(self, min: Self, max: Self) -> Self { - assert!( - min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - self.simd_max(min).simd_min(max) - } - } - )* - } -} - -impl_integer! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } - -macro_rules! impl_float { - { $($float:ty),* } => { - $( - impl SimdPartialOrd for Simd<$float, LANES> - where - LaneCount: SupportedLaneCount, - { - #[inline] - fn simd_lt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } - } - - #[inline] - fn simd_le(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } - } - - #[inline] - fn simd_gt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } - } - - #[inline] - fn simd_ge(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } - } - } - )* - } -} - -impl_float! { f32, f64 } - -macro_rules! impl_mask { - { $($integer:ty),* } => { - $( - impl SimdPartialOrd for Mask<$integer, LANES> - where - LaneCount: SupportedLaneCount, - { - #[inline] - fn simd_lt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) } - } - - #[inline] - fn simd_le(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) } - } - - #[inline] - fn simd_gt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) } - } - - #[inline] - fn simd_ge(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) } - } - } - - impl SimdOrd for Mask<$integer, LANES> - where - LaneCount: SupportedLaneCount, - { - #[inline] - fn simd_max(self, other: Self) -> Self { - self.simd_gt(other).select_mask(other, self) - } - - #[inline] - fn simd_min(self, other: Self) -> Self { - self.simd_lt(other).select_mask(other, self) - } - - #[inline] - #[track_caller] - fn simd_clamp(self, min: Self, max: Self) -> Self { - assert!( - min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - self.simd_max(min).simd_min(max) - } - } - )* - } -} - -impl_mask! { i8, i16, i32, i64, isize } - -impl SimdPartialOrd for Simd<*const T, LANES> -where - LaneCount: SupportedLaneCount, -{ - #[inline] - fn simd_lt(self, other: Self) -> Self::Mask { - self.addr().simd_lt(other.addr()) - } - - #[inline] - fn simd_le(self, other: Self) -> Self::Mask { - self.addr().simd_le(other.addr()) - } - - #[inline] - fn simd_gt(self, other: Self) -> Self::Mask { - self.addr().simd_gt(other.addr()) - } - - #[inline] - fn simd_ge(self, other: Self) -> Self::Mask { - self.addr().simd_ge(other.addr()) - } -} - -impl SimdOrd for Simd<*const T, LANES> -where - LaneCount: SupportedLaneCount, -{ - #[inline] - fn simd_max(self, other: Self) -> Self { - self.simd_lt(other).select(other, self) - } - - #[inline] - fn simd_min(self, other: Self) -> Self { - self.simd_gt(other).select(other, self) - } - - #[inline] - #[track_caller] - fn simd_clamp(self, min: Self, max: Self) -> Self { - assert!( - min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - self.simd_max(min).simd_min(max) - } -} - -impl SimdPartialOrd for Simd<*mut T, LANES> -where - LaneCount: SupportedLaneCount, -{ - #[inline] - fn simd_lt(self, other: Self) -> Self::Mask { - self.addr().simd_lt(other.addr()) - } - - #[inline] - fn simd_le(self, other: Self) -> Self::Mask { - self.addr().simd_le(other.addr()) - } - - #[inline] - fn simd_gt(self, other: Self) -> Self::Mask { - self.addr().simd_gt(other.addr()) - } - - #[inline] - fn simd_ge(self, other: Self) -> Self::Mask { - self.addr().simd_ge(other.addr()) - } -} - -impl SimdOrd for Simd<*mut T, LANES> -where - LaneCount: SupportedLaneCount, -{ - #[inline] - fn simd_max(self, other: Self) -> Self { - self.simd_lt(other).select(other, self) - } - - #[inline] - fn simd_min(self, other: Self) -> Self { - self.simd_gt(other).select(other, self) - } - - #[inline] - #[track_caller] - fn simd_clamp(self, min: Self, max: Self) -> Self { - assert!( - min.simd_le(max).all(), - "each lane in `min` must be less than or equal to the corresponding lane in `max`", - ); - self.simd_max(min).simd_min(max) - } -} diff --git a/crates/core_simd/src/simd/cmp.rs b/crates/core_simd/src/simd/cmp.rs new file mode 100644 index 00000000000..a8d81dbf20f --- /dev/null +++ b/crates/core_simd/src/simd/cmp.rs @@ -0,0 +1,7 @@ +//! Traits for comparing and ordering vectors. + +mod eq; +mod ord; + +pub use eq::*; +pub use ord::*; diff --git a/crates/core_simd/src/simd/cmp/eq.rs b/crates/core_simd/src/simd/cmp/eq.rs new file mode 100644 index 00000000000..627ceba3c6f --- /dev/null +++ b/crates/core_simd/src/simd/cmp/eq.rs @@ -0,0 +1,111 @@ +use crate::simd::{ + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, Simd, SimdElement, SupportedLaneCount, +}; + +/// Parallel `PartialEq`. +pub trait SimdPartialEq { + /// The mask type returned by each comparison. + type Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_eq(self, other: Self) -> Self::Mask; + + /// Test if each lane is equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ne(self, other: Self) -> Self::Mask; +} + +macro_rules! impl_number { + { $($number:ty),* } => { + $( + impl SimdPartialEq for Simd<$number, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$number as SimdElement>::Mask, LANES>; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } + } + )* + } +} + +impl_number! { f32, f64, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialEq for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Self; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) } + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialEq for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + self.addr().simd_eq(other.addr()) + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + self.addr().simd_ne(other.addr()) + } +} + +impl SimdPartialEq for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + self.addr().simd_eq(other.addr()) + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + self.addr().simd_ne(other.addr()) + } +} diff --git a/crates/core_simd/src/simd/cmp/ord.rs b/crates/core_simd/src/simd/cmp/ord.rs new file mode 100644 index 00000000000..509f907785c --- /dev/null +++ b/crates/core_simd/src/simd/cmp/ord.rs @@ -0,0 +1,320 @@ +use crate::simd::{ + cmp::SimdPartialEq, + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, Simd, SupportedLaneCount, +}; + +/// Parallel `PartialOrd`. +pub trait SimdPartialOrd: SimdPartialEq { + /// Test if each lane is less than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_lt(self, other: Self) -> Self::Mask; + + /// Test if each lane is less than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_le(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_gt(self, other: Self) -> Self::Mask; + + /// Test if each lane is greater than or equal to the corresponding lane in `other`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn simd_ge(self, other: Self) -> Self::Mask; +} + +/// Parallel `Ord`. +pub trait SimdOrd: SimdPartialOrd { + /// Returns the lane-wise maximum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Returns the lane-wise minimum with `other`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval. + /// + /// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is + /// less than `min`. Otherwise returns `self`. + /// + /// # Panics + /// + /// Panics if `min > max` on any lane. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; +} + +macro_rules! impl_integer { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + + impl SimdOrd for Simd<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + #[track_caller] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_integer! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize } + +macro_rules! impl_float { + { $($float:ty),* } => { + $( + impl SimdPartialOrd for Simd<$float, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } + } + )* + } +} + +impl_float! { f32, f64 } + +macro_rules! impl_mask { + { $($integer:ty),* } => { + $( + impl SimdPartialOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) } + } + } + + impl SimdOrd for Mask<$integer, LANES> + where + LaneCount: SupportedLaneCount, + { + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_gt(other).select_mask(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_lt(other).select_mask(other, self) + } + + #[inline] + #[track_caller] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } + } + )* + } +} + +impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + self.addr().simd_lt(other.addr()) + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + self.addr().simd_le(other.addr()) + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + self.addr().simd_gt(other.addr()) + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + self.addr().simd_ge(other.addr()) + } +} + +impl SimdOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + #[track_caller] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} + +impl SimdPartialOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + self.addr().simd_lt(other.addr()) + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + self.addr().simd_le(other.addr()) + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + self.addr().simd_gt(other.addr()) + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + self.addr().simd_ge(other.addr()) + } +} + +impl SimdOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + #[track_caller] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} diff --git a/crates/core_simd/src/simd/num.rs b/crates/core_simd/src/simd/num.rs new file mode 100644 index 00000000000..22a4802ec6c --- /dev/null +++ b/crates/core_simd/src/simd/num.rs @@ -0,0 +1,13 @@ +//! Traits for vectors with numeric elements. + +mod float; +mod int; +mod uint; + +mod sealed { + pub trait Sealed {} +} + +pub use float::*; +pub use int::*; +pub use uint::*; diff --git a/crates/core_simd/src/simd/num/float.rs b/crates/core_simd/src/simd/num/float.rs new file mode 100644 index 00000000000..affc01d111f --- /dev/null +++ b/crates/core_simd/src/simd/num/float.rs @@ -0,0 +1,425 @@ +use super::sealed::Sealed; +use crate::simd::{ + cmp::{SimdPartialEq, SimdPartialOrd}, + intrinsics, LaneCount, Mask, Simd, SimdCast, SimdElement, SupportedLaneCount, +}; + +/// Operations on SIMD vectors of floats. +pub trait SimdFloat: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// Bit representation of this SIMD vector type. + type Bits; + + /// A SIMD vector with a different element type. + type Cast; + + /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. + /// + /// This follows the semantics of Rust's `as` conversion for floats (truncating or saturating + /// at the limits) for each element. + /// + /// # Example + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let floats: Simd = Simd::from_array([1.9, -4.5, f32::INFINITY, f32::NAN]); + /// let ints = floats.cast::(); + /// assert_eq!(ints, Simd::from_array([1, -4, i32::MAX, 0])); + /// + /// // Formally equivalent, but `Simd::cast` can optimize better. + /// assert_eq!(ints, Simd::from_array(floats.to_array().map(|x| x as i32))); + /// + /// // The float conversion does not round-trip. + /// let floats_again = ints.cast(); + /// assert_ne!(floats, floats_again); + /// assert_eq!(floats_again, Simd::from_array([1.0, -4.0, 2147483647.0, 0.0])); + /// ``` + #[must_use] + fn cast(self) -> Self::Cast; + + /// Rounds toward zero and converts to the same-width integer type, assuming that + /// the value is finite and fits in that type. + /// + /// # Safety + /// The value must: + /// + /// * Not be NaN + /// * Not be infinite + /// * Be representable in the return type, after truncating off its fractional part + /// + /// If these requirements are infeasible or costly, consider using the safe function [cast], + /// which saturates on conversion. + /// + /// [cast]: Simd::cast + unsafe fn to_int_unchecked(self) -> Self::Cast + where + Self::Scalar: core::convert::FloatToInt; + + /// Raw transmutation to an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_bits(self) -> Self::Bits; + + /// Raw transmutation from an unsigned integer vector type with the + /// same size and number of lanes. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn from_bits(bits: Self::Bits) -> Self; + + /// Produces a vector where every lane has the absolute value of the + /// equivalently-indexed lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn abs(self) -> Self; + + /// Takes the reciprocal (inverse) of each lane, `1/x`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn recip(self) -> Self; + + /// Converts each lane from radians to degrees. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_degrees(self) -> Self; + + /// Converts each lane from degrees to radians. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn to_radians(self) -> Self; + + /// Returns true for each lane if it has a positive sign, including + /// `+0.0`, `NaN`s with positive sign bit and positive infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_positive(self) -> Self::Mask; + + /// Returns true for each lane if it has a negative sign, including + /// `-0.0`, `NaN`s with negative sign bit and negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_sign_negative(self) -> Self::Mask; + + /// Returns true for each lane if its value is `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_nan(self) -> Self::Mask; + + /// Returns true for each lane if its value is positive infinity or negative infinity. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_infinite(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither infinite nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_finite(self) -> Self::Mask; + + /// Returns true for each lane if its value is subnormal. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_subnormal(self) -> Self::Mask; + + /// Returns true for each lane if its value is neither zero, infinite, + /// subnormal, nor `NaN`. + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask; + + /// Replaces each lane with a number that represents its sign. + /// + /// * `1.0` if the number is positive, `+0.0`, or `INFINITY` + /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY` + /// * `NAN` if the number is `NAN` + #[must_use = "method returns a new vector and does not mutate the original value"] + fn signum(self) -> Self; + + /// Returns each lane with the magnitude of `self` and the sign of `sign`. + /// + /// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn copysign(self, sign: Self) -> Self; + + /// Returns the minimum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_min(self, other: Self) -> Self; + + /// Returns the maximum of each lane. + /// + /// If one of the values is `NAN`, then the other value is returned. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_max(self, other: Self) -> Self; + + /// Restrict each lane to a certain interval unless it is NaN. + /// + /// For each lane in `self`, returns the corresponding lane in `max` if the lane is + /// greater than `max`, and the corresponding lane in `min` if the lane is less + /// than `min`. Otherwise returns the lane in `self`. + #[must_use = "method returns a new vector and does not mutate the original value"] + fn simd_clamp(self, min: Self, max: Self) -> Self; + + /// Returns the sum of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_sum(), 3.); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Reducing multiply. Returns the product of the lanes of the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = f32x2::from_array([3., 4.]); + /// assert_eq!(v.reduce_product(), 12.); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = f32x2::from_array([1., 2.]); + /// assert_eq!(v.reduce_max(), 2.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_max(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_max().is_nan()); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. + /// + /// This function will not return `NaN` unless all lanes are `NaN`. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = f32x2::from_array([3., 7.]); + /// assert_eq!(v.reduce_min(), 3.); + /// + /// // NaN values are skipped... + /// let v = f32x2::from_array([1., f32::NAN]); + /// assert_eq!(v.reduce_min(), 1.); + /// + /// // ...unless all values are NaN + /// let v = f32x2::from_array([f32::NAN, f32::NAN]); + /// assert!(v.reduce_min().is_nan()); + /// ``` + fn reduce_min(self) -> Self::Scalar; +} + +macro_rules! impl_trait { + { $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdFloat for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + type Bits = Simd<$bits_ty, LANES>; + type Cast = Simd; + + #[inline] + fn cast(self) -> Self::Cast + { + // Safety: supported types are guaranteed by SimdCast + unsafe { intrinsics::simd_as(self) } + } + + #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + unsafe fn to_int_unchecked(self) -> Self::Cast + where + Self::Scalar: core::convert::FloatToInt, + { + // Safety: supported types are guaranteed by SimdCast, the caller is responsible for the extra invariants + unsafe { intrinsics::simd_cast(self) } + } + + #[inline] + fn to_bits(self) -> Simd<$bits_ty, LANES> { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self { + assert_eq!(core::mem::size_of::(), core::mem::size_of::()); + // Safety: transmuting between vector types is safe + unsafe { core::mem::transmute_copy(&bits) } + } + + #[inline] + fn abs(self) -> Self { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_fabs(self) } + } + + #[inline] + fn recip(self) -> Self { + Self::splat(1.0) / self + } + + #[inline] + fn to_degrees(self) -> Self { + // to_degrees uses a special constant for better precision, so extract that constant + self * Self::splat(Self::Scalar::to_degrees(1.)) + } + + #[inline] + fn to_radians(self) -> Self { + self * Self::splat(Self::Scalar::to_radians(1.)) + } + + #[inline] + fn is_sign_positive(self) -> Self::Mask { + !self.is_sign_negative() + } + + #[inline] + fn is_sign_negative(self) -> Self::Mask { + let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1); + sign_bits.simd_gt(Simd::splat(0)) + } + + #[inline] + fn is_nan(self) -> Self::Mask { + self.simd_ne(self) + } + + #[inline] + fn is_infinite(self) -> Self::Mask { + self.abs().simd_eq(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_finite(self) -> Self::Mask { + self.abs().simd_lt(Self::splat(Self::Scalar::INFINITY)) + } + + #[inline] + fn is_subnormal(self) -> Self::Mask { + // On some architectures (e.g. armv7 and some ppc) subnormals are flushed to zero, + // so this comparison must be done with integers. + let not_zero = self.abs().to_bits().simd_ne(Self::splat(0.0).to_bits()); + not_zero & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0)) + } + + #[inline] + #[must_use = "method returns a new mask and does not mutate the original value"] + fn is_normal(self) -> Self::Mask { + !(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite()) + } + + #[inline] + fn signum(self) -> Self { + self.is_nan().select(Self::splat(Self::Scalar::NAN), Self::splat(1.0).copysign(self)) + } + + #[inline] + fn copysign(self, sign: Self) -> Self { + let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits(); + let magnitude = self.to_bits() & !Self::splat(-0.).to_bits(); + Self::from_bits(sign_bit | magnitude) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + // Safety: `self` and `other` are float vectors + unsafe { intrinsics::simd_fmin(self, other) } + } + + #[inline] + fn simd_max(self, other: Self) -> Self { + // Safety: `self` and `other` are floating point vectors + unsafe { intrinsics::simd_fmax(self, other) } + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + let mut x = self; + x = x.simd_lt(min).select(min, x); + x = x.simd_gt(max).select(max, x); + x + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // LLVM sum is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().sum() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) } + } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // LLVM product is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_array().iter().product() + } else { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) } + } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is a float vector + unsafe { intrinsics::simd_reduce_min(self) } + } + } + )* + } +} + +impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } diff --git a/crates/core_simd/src/simd/num/int.rs b/crates/core_simd/src/simd/num/int.rs new file mode 100644 index 00000000000..d1f8e856a53 --- /dev/null +++ b/crates/core_simd/src/simd/num/int.rs @@ -0,0 +1,371 @@ +use super::sealed::Sealed; +use crate::simd::{ + cmp::SimdPartialOrd, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SimdCast, SimdElement, + SupportedLaneCount, +}; + +/// Operations on SIMD vectors of signed integers. +pub trait SimdInt: Copy + Sealed { + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// A SIMD vector of unsigned integers with the same element size. + type Unsigned; + + /// A SIMD vector with a different element type. + type Cast; + + /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. + /// + /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to + /// other integer types, and saturating to float types). + #[must_use] + fn cast(self) -> Self::Cast; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, 0, 1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2])); + /// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX])); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, -1, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0])); + /// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0])); + fn saturating_sub(self, second: Self) -> Self; + + /// Lanewise absolute value, implemented in Rust. + /// Every lane becomes its absolute value. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, MIN +1, -5, 0]); + /// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0])); + /// ``` + fn abs(self) -> Self; + + /// Lanewise saturating absolute value, implemented in Rust. + /// As abs(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let xs = Simd::from_array([MIN, -2, 0, 3]); + /// let unsat = xs.abs(); + /// let sat = xs.saturating_abs(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3])); + /// ``` + fn saturating_abs(self) -> Self; + + /// Lanewise saturating negation, implemented in Rust. + /// As neg(), except the MIN value becomes MAX instead of itself. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::i32::{MIN, MAX}; + /// let x = Simd::from_array([MIN, -2, 3, MAX]); + /// let unsat = -x; + /// let sat = x.saturating_neg(); + /// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1])); + /// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1])); + /// ``` + fn saturating_neg(self) -> Self; + + /// Returns true for each positive lane and false if it is zero or negative. + fn is_positive(self) -> Self::Mask; + + /// Returns true for each negative lane and false if it is zero or positive. + fn is_negative(self) -> Self::Mask; + + /// Returns numbers representing the sign of each lane. + /// * `0` if the number is zero + /// * `1` if the number is positive + /// * `-1` if the number is negative + fn signum(self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_sum(), 10); + /// + /// // SIMD integer addition is always wrapping + /// let v = i32x4::from_array([i32::MAX, 1, 0, 0]); + /// assert_eq!(v.reduce_sum(), i32::MIN); + /// ``` + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_product(), 24); + /// + /// // SIMD integer multiplication is always wrapping + /// let v = i32x4::from_array([i32::MAX, 2, 1, 1]); + /// assert!(v.reduce_product() < i32::MAX); + /// ``` + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_max(), 4); + /// ``` + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + /// + /// # Examples + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// let v = i32x4::from_array([1, 2, 3, 4]); + /// assert_eq!(v.reduce_min(), 1); + /// ``` + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; + + /// Reverses the byte order of each element. + fn swap_bytes(self) -> Self; + + /// Reverses the order of bits in each elemnent. + /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. + fn reverse_bits(self) -> Self; + + /// Returns the number of leading zeros in the binary representation of each element. + fn leading_zeros(self) -> Self::Unsigned; + + /// Returns the number of trailing zeros in the binary representation of each element. + fn trailing_zeros(self) -> Self::Unsigned; + + /// Returns the number of leading ones in the binary representation of each element. + fn leading_ones(self) -> Self::Unsigned; + + /// Returns the number of trailing ones in the binary representation of each element. + fn trailing_ones(self) -> Self::Unsigned; +} + +macro_rules! impl_trait { + { $($ty:ident ($unsigned:ident)),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdInt for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Mask = Mask<<$ty as SimdElement>::Mask, LANES>; + type Scalar = $ty; + type Unsigned = Simd<$unsigned, LANES>; + type Cast = Simd; + + #[inline] + fn cast(self) -> Self::Cast { + // Safety: supported types are guaranteed by SimdCast + unsafe { intrinsics::simd_as(self) } + } + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn abs(self) -> Self { + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m) - m + } + + #[inline] + fn saturating_abs(self) -> Self { + // arith shift for -1 or 0 mask based on sign bit, giving 2s complement + const SHR: $ty = <$ty>::BITS as $ty - 1; + let m = self >> Simd::splat(SHR); + (self^m).saturating_sub(m) + } + + #[inline] + fn saturating_neg(self) -> Self { + Self::splat(0).saturating_sub(self) + } + + #[inline] + fn is_positive(self) -> Self::Mask { + self.simd_gt(Self::splat(0)) + } + + #[inline] + fn is_negative(self) -> Self::Mask { + self.simd_lt(Self::splat(0)) + } + + #[inline] + fn signum(self) -> Self { + self.is_positive().select( + Self::splat(1), + self.is_negative().select(Self::splat(-1), Self::splat(0)) + ) + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + + #[inline] + fn swap_bytes(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bswap(self) } + } + + #[inline] + fn reverse_bits(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bitreverse(self) } + } + + #[inline] + fn leading_zeros(self) -> Self::Unsigned { + self.cast::<$unsigned>().leading_zeros() + } + + #[inline] + fn trailing_zeros(self) -> Self::Unsigned { + self.cast::<$unsigned>().trailing_zeros() + } + + #[inline] + fn leading_ones(self) -> Self::Unsigned { + self.cast::<$unsigned>().leading_ones() + } + + #[inline] + fn trailing_ones(self) -> Self::Unsigned { + self.cast::<$unsigned>().trailing_ones() + } + } + )* + } +} + +impl_trait! { i8 (u8), i16 (u16), i32 (u32), i64 (u64), isize (usize) } diff --git a/crates/core_simd/src/simd/num/uint.rs b/crates/core_simd/src/simd/num/uint.rs new file mode 100644 index 00000000000..7eadd2050b9 --- /dev/null +++ b/crates/core_simd/src/simd/num/uint.rs @@ -0,0 +1,221 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Simd, SimdCast, SimdElement, SupportedLaneCount}; + +/// Operations on SIMD vectors of unsigned integers. +pub trait SimdUint: Copy + Sealed { + /// Scalar type contained by this SIMD vector type. + type Scalar; + + /// A SIMD vector with a different element type. + type Cast; + + /// Performs elementwise conversion of this vector's elements to another SIMD-valid type. + /// + /// This follows the semantics of Rust's `as` conversion for casting integers (wrapping to + /// other integer types, and saturating to float types). + #[must_use] + fn cast(self) -> Self::Cast; + + /// Wrapping negation. + /// + /// Like [`u32::wrapping_neg`], all applications of this function will wrap, with the exception + /// of `-0`. + fn wrapping_neg(self) -> Self; + + /// Lanewise saturating add. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x + max; + /// let sat = x.saturating_add(max); + /// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1])); + /// assert_eq!(sat, max); + /// ``` + fn saturating_add(self, second: Self) -> Self; + + /// Lanewise saturating subtract. + /// + /// # Examples + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::prelude::*; + /// use core::u32::MAX; + /// let x = Simd::from_array([2, 1, 0, MAX]); + /// let max = Simd::splat(MAX); + /// let unsat = x - max; + /// let sat = x.saturating_sub(max); + /// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0])); + /// assert_eq!(sat, Simd::splat(0)); + fn saturating_sub(self, second: Self) -> Self; + + /// Returns the sum of the lanes of the vector, with wrapping addition. + fn reduce_sum(self) -> Self::Scalar; + + /// Returns the product of the lanes of the vector, with wrapping multiplication. + fn reduce_product(self) -> Self::Scalar; + + /// Returns the maximum lane in the vector. + fn reduce_max(self) -> Self::Scalar; + + /// Returns the minimum lane in the vector. + fn reduce_min(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "and" across the lanes of the vector. + fn reduce_and(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "or" across the lanes of the vector. + fn reduce_or(self) -> Self::Scalar; + + /// Returns the cumulative bitwise "xor" across the lanes of the vector. + fn reduce_xor(self) -> Self::Scalar; + + /// Reverses the byte order of each element. + fn swap_bytes(self) -> Self; + + /// Reverses the order of bits in each elemnent. + /// The least significant bit becomes the most significant bit, second least-significant bit becomes second most-significant bit, etc. + fn reverse_bits(self) -> Self; + + /// Returns the number of leading zeros in the binary representation of each element. + fn leading_zeros(self) -> Self; + + /// Returns the number of trailing zeros in the binary representation of each element. + fn trailing_zeros(self) -> Self; + + /// Returns the number of leading ones in the binary representation of each element. + fn leading_ones(self) -> Self; + + /// Returns the number of trailing ones in the binary representation of each element. + fn trailing_ones(self) -> Self; +} + +macro_rules! impl_trait { + { $($ty:ident ($signed:ident)),* } => { + $( + impl Sealed for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + } + + impl SimdUint for Simd<$ty, LANES> + where + LaneCount: SupportedLaneCount, + { + type Scalar = $ty; + type Cast = Simd; + + #[inline] + fn cast(self) -> Self::Cast { + // Safety: supported types are guaranteed by SimdCast + unsafe { intrinsics::simd_as(self) } + } + + #[inline] + fn wrapping_neg(self) -> Self { + use crate::simd::num::SimdInt; + (-self.cast::<$signed>()).cast() + } + + #[inline] + fn saturating_add(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_add(self, second) } + } + + #[inline] + fn saturating_sub(self, second: Self) -> Self { + // Safety: `self` is a vector + unsafe { intrinsics::simd_saturating_sub(self, second) } + } + + #[inline] + fn reduce_sum(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_add_ordered(self, 0) } + } + + #[inline] + fn reduce_product(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + #[inline] + fn reduce_max(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_max(self) } + } + + #[inline] + fn reduce_min(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_min(self) } + } + + #[inline] + fn reduce_and(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_and(self) } + } + + #[inline] + fn reduce_or(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_or(self) } + } + + #[inline] + fn reduce_xor(self) -> Self::Scalar { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_reduce_xor(self) } + } + + #[inline] + fn swap_bytes(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bswap(self) } + } + + #[inline] + fn reverse_bits(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_bitreverse(self) } + } + + #[inline] + fn leading_zeros(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_ctlz(self) } + } + + #[inline] + fn trailing_zeros(self) -> Self { + // Safety: `self` is an integer vector + unsafe { intrinsics::simd_cttz(self) } + } + + #[inline] + fn leading_ones(self) -> Self { + (!self).leading_zeros() + } + + #[inline] + fn trailing_ones(self) -> Self { + (!self).trailing_zeros() + } + } + )* + } +} + +impl_trait! { u8 (i8), u16 (i16), u32 (i32), u64 (i64), usize (isize) } diff --git a/crates/core_simd/src/simd/prelude.rs b/crates/core_simd/src/simd/prelude.rs index e8fdc932d49..4b7c744c013 100644 --- a/crates/core_simd/src/simd/prelude.rs +++ b/crates/core_simd/src/simd/prelude.rs @@ -7,8 +7,10 @@ #[doc(no_inline)] pub use super::{ - simd_swizzle, Mask, Simd, SimdConstPtr, SimdFloat, SimdInt, SimdMutPtr, SimdOrd, SimdPartialEq, - SimdPartialOrd, SimdUint, + cmp::{SimdOrd, SimdPartialEq, SimdPartialOrd}, + num::{SimdFloat, SimdInt, SimdUint}, + ptr::{SimdConstPtr, SimdMutPtr}, + simd_swizzle, Mask, Simd, }; #[rustfmt::skip] diff --git a/crates/core_simd/src/simd/ptr.rs b/crates/core_simd/src/simd/ptr.rs new file mode 100644 index 00000000000..3f8e6669118 --- /dev/null +++ b/crates/core_simd/src/simd/ptr.rs @@ -0,0 +1,11 @@ +//! Traits for vectors of pointers. + +mod const_ptr; +mod mut_ptr; + +mod sealed { + pub trait Sealed {} +} + +pub use const_ptr::*; +pub use mut_ptr::*; diff --git a/crates/core_simd/src/simd/ptr/const_ptr.rs b/crates/core_simd/src/simd/ptr/const_ptr.rs new file mode 100644 index 00000000000..f82def1d377 --- /dev/null +++ b/crates/core_simd/src/simd/ptr/const_ptr.rs @@ -0,0 +1,165 @@ +use super::sealed::Sealed; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount, +}; + +/// Operations on SIMD vectors of constant pointers. +pub trait SimdConstPtr: Copy + Sealed { + /// Vector of `usize` with the same number of lanes. + type Usize; + + /// Vector of `isize` with the same number of lanes. + type Isize; + + /// Vector of const pointers with the same number of lanes. + type CastPtr; + + /// Vector of mutable pointers to the same type. + type MutPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Casts to a pointer of another type. + /// + /// Equivalent to calling [`pointer::cast`] on each lane. + fn cast(self) -> Self::CastPtr; + + /// Changes constness without changing the type. + /// + /// Equivalent to calling [`pointer::cast_mut`] on each lane. + fn cast_mut(self) -> Self::MutPtr; + + /// Gets the "address" portion of the pointer. + /// + /// This method discards pointer semantic metadata, so the result cannot be + /// directly cast into a valid pointer. + /// + /// This method semantically discards *provenance* and + /// *address-space* information. To properly restore that information, use [`Self::with_addr`]. + /// + /// Equivalent to calling [`pointer::addr`] on each lane. + fn addr(self) -> Self::Usize; + + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as a cast, but copies the *address-space* and + /// *provenance* of `self` to the new pointer. + /// + /// Equivalent to calling [`pointer::with_addr`] on each lane. + fn with_addr(self, addr: Self::Usize) -> Self; + + /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use + /// in [`Self::from_exposed_addr`]. + fn expose_addr(self) -> Self::Usize; + + /// Convert an address back to a pointer, picking up a previously "exposed" provenance. + /// + /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each lane. + fn from_exposed_addr(addr: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + fn wrapping_offset(self, offset: Self::Isize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_add(self, count: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. + fn wrapping_sub(self, count: Self::Usize) -> Self; +} + +impl Sealed for Simd<*const T, LANES> where + LaneCount: SupportedLaneCount +{ +} + +impl SimdConstPtr for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Usize = Simd; + type Isize = Simd; + type CastPtr = Simd<*const U, LANES>; + type MutPtr = Simd<*mut T, LANES>; + type Mask = Mask; + + #[inline] + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null()).simd_eq(self) + } + + #[inline] + fn cast(self) -> Self::CastPtr { + // SimdElement currently requires zero-sized metadata, so this should never fail. + // If this ever changes, `simd_cast_ptr` should produce a post-mono error. + use core::{mem::size_of, ptr::Pointee}; + assert_eq!(size_of::<::Metadata>(), 0); + assert_eq!(size_of::<::Metadata>(), 0); + + // Safety: pointers can be cast + unsafe { intrinsics::simd_cast_ptr(self) } + } + + #[inline] + fn cast_mut(self) -> Self::MutPtr { + // Safety: pointers can be cast + unsafe { intrinsics::simd_cast_ptr(self) } + } + + #[inline] + fn addr(self) -> Self::Usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + fn with_addr(self, addr: Self::Usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. + self.cast::() + .wrapping_offset(addr.cast::() - self.addr().cast::()) + .cast() + } + + #[inline] + fn expose_addr(self) -> Self::Usize { + // Safety: `self` is a pointer vector + unsafe { intrinsics::simd_expose_addr(self) } + } + + #[inline] + fn from_exposed_addr(addr: Self::Usize) -> Self { + // Safety: `self` is a pointer vector + unsafe { intrinsics::simd_from_exposed_addr(addr) } + } + + #[inline] + fn wrapping_offset(self, count: Self::Isize) -> Self { + // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets + unsafe { intrinsics::simd_arith_offset(self, count) } + } + + #[inline] + fn wrapping_add(self, count: Self::Usize) -> Self { + self.wrapping_offset(count.cast()) + } + + #[inline] + fn wrapping_sub(self, count: Self::Usize) -> Self { + self.wrapping_offset(-count.cast::()) + } +} diff --git a/crates/core_simd/src/simd/ptr/mut_ptr.rs b/crates/core_simd/src/simd/ptr/mut_ptr.rs new file mode 100644 index 00000000000..283054dc8ce --- /dev/null +++ b/crates/core_simd/src/simd/ptr/mut_ptr.rs @@ -0,0 +1,160 @@ +use super::sealed::Sealed; +use crate::simd::{ + cmp::SimdPartialEq, intrinsics, num::SimdUint, LaneCount, Mask, Simd, SupportedLaneCount, +}; + +/// Operations on SIMD vectors of mutable pointers. +pub trait SimdMutPtr: Copy + Sealed { + /// Vector of `usize` with the same number of lanes. + type Usize; + + /// Vector of `isize` with the same number of lanes. + type Isize; + + /// Vector of const pointers with the same number of lanes. + type CastPtr; + + /// Vector of constant pointers to the same type. + type ConstPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Casts to a pointer of another type. + /// + /// Equivalent to calling [`pointer::cast`] on each lane. + fn cast(self) -> Self::CastPtr; + + /// Changes constness without changing the type. + /// + /// Equivalent to calling [`pointer::cast_const`] on each lane. + fn cast_const(self) -> Self::ConstPtr; + + /// Gets the "address" portion of the pointer. + /// + /// This method discards pointer semantic metadata, so the result cannot be + /// directly cast into a valid pointer. + /// + /// Equivalent to calling [`pointer::addr`] on each lane. + fn addr(self) -> Self::Usize; + + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as a cast, but copies the *address-space* and + /// *provenance* of `self` to the new pointer. + /// + /// Equivalent to calling [`pointer::with_addr`] on each lane. + fn with_addr(self, addr: Self::Usize) -> Self; + + /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use + /// in [`Self::from_exposed_addr`]. + fn expose_addr(self) -> Self::Usize; + + /// Convert an address back to a pointer, picking up a previously "exposed" provenance. + /// + /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each lane. + fn from_exposed_addr(addr: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + fn wrapping_offset(self, offset: Self::Isize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_add(self, count: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. + fn wrapping_sub(self, count: Self::Usize) -> Self; +} + +impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount +{} + +impl SimdMutPtr for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Usize = Simd; + type Isize = Simd; + type CastPtr = Simd<*mut U, LANES>; + type ConstPtr = Simd<*const T, LANES>; + type Mask = Mask; + + #[inline] + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null_mut()).simd_eq(self) + } + + #[inline] + fn cast(self) -> Self::CastPtr { + // SimdElement currently requires zero-sized metadata, so this should never fail. + // If this ever changes, `simd_cast_ptr` should produce a post-mono error. + use core::{mem::size_of, ptr::Pointee}; + assert_eq!(size_of::<::Metadata>(), 0); + assert_eq!(size_of::<::Metadata>(), 0); + + // Safety: pointers can be cast + unsafe { intrinsics::simd_cast_ptr(self) } + } + + #[inline] + fn cast_const(self) -> Self::ConstPtr { + // Safety: pointers can be cast + unsafe { intrinsics::simd_cast_ptr(self) } + } + + #[inline] + fn addr(self) -> Self::Usize { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). + unsafe { core::mem::transmute_copy(&self) } + } + + #[inline] + fn with_addr(self, addr: Self::Usize) -> Self { + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. + self.cast::() + .wrapping_offset(addr.cast::() - self.addr().cast::()) + .cast() + } + + #[inline] + fn expose_addr(self) -> Self::Usize { + // Safety: `self` is a pointer vector + unsafe { intrinsics::simd_expose_addr(self) } + } + + #[inline] + fn from_exposed_addr(addr: Self::Usize) -> Self { + // Safety: `self` is a pointer vector + unsafe { intrinsics::simd_from_exposed_addr(addr) } + } + + #[inline] + fn wrapping_offset(self, count: Self::Isize) -> Self { + // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets + unsafe { intrinsics::simd_arith_offset(self, count) } + } + + #[inline] + fn wrapping_add(self, count: Self::Usize) -> Self { + self.wrapping_offset(count.cast()) + } + + #[inline] + fn wrapping_sub(self, count: Self::Usize) -> Self { + self.wrapping_offset(-count.cast::()) + } +} diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index ce621792534..bd8a38e350d 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -86,7 +86,7 @@ where #[inline] #[allow(clippy::let_and_return)] unsafe fn avx2_pshufb(bytes: Simd, idxs: Simd) -> Simd { - use crate::simd::SimdPartialOrd; + use crate::simd::cmp::SimdPartialOrd; #[cfg(target_arch = "x86")] use core::arch::x86; #[cfg(target_arch = "x86_64")] @@ -149,7 +149,7 @@ where // On x86, make sure the top bit is set. #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] let idxs = { - use crate::simd::SimdPartialOrd; + use crate::simd::cmp::SimdPartialOrd; idxs.simd_lt(Simd::splat(N as u8)) .select(idxs, Simd::splat(u8::MAX)) }; diff --git a/crates/core_simd/src/to_bytes.rs b/crates/core_simd/src/to_bytes.rs index 3c93fe47404..dd01929551c 100644 --- a/crates/core_simd/src/to_bytes.rs +++ b/crates/core_simd/src/to_bytes.rs @@ -1,4 +1,7 @@ -use crate::simd::{LaneCount, Simd, SimdElement, SimdFloat, SimdInt, SimdUint, SupportedLaneCount}; +use crate::simd::{ + num::{SimdFloat, SimdInt, SimdUint}, + LaneCount, Simd, SimdElement, SupportedLaneCount, +}; mod sealed { use super::*; diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 9aa7bacfce9..70188337444 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,6 +1,8 @@ use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdConstPtr, SimdMutPtr, SimdPartialOrd, - SupportedLaneCount, Swizzle, + cmp::SimdPartialOrd, + intrinsics, + ptr::{SimdConstPtr, SimdMutPtr}, + LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle, }; use core::convert::{TryFrom, TryInto}; @@ -394,7 +396,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// # use simd::{Simd, cmp::SimdPartialOrd, Mask}; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index /// let alt = Simd::from_array([-5, -4, -3, -2]); @@ -434,7 +436,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdConstPtr}; + /// # use simd::prelude::*; /// let values = [6, 2, 4, 9]; /// let offsets = Simd::from_array([1, 0, 0, 3]); /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets); @@ -467,7 +469,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Mask, Simd, SimdConstPtr}; + /// # use simd::prelude::*; /// let values = [6, 2, 4, 9]; /// let enable = Mask::from_array([true, true, false, true]); /// let offsets = Simd::from_array([1, 0, 0, 3]); @@ -550,7 +552,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdPartialOrd, Mask}; + /// # use simd::{Simd, cmp::SimdPartialOrd, Mask}; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 0]); /// let vals = Simd::from_array([-27, 82, -41, 124]); @@ -604,7 +606,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Simd, SimdMutPtr}; + /// # use simd::{Simd, ptr::SimdMutPtr}; /// let mut values = [0; 4]; /// let offset = Simd::from_array([3, 2, 1, 0]); /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); @@ -631,7 +633,7 @@ where /// # #![feature(portable_simd)] /// # #[cfg(feature = "as_crate")] use core_simd::simd; /// # #[cfg(not(feature = "as_crate"))] use core::simd; - /// # use simd::{Mask, Simd, SimdMutPtr}; + /// # use simd::{Mask, Simd, ptr::SimdMutPtr}; /// let mut values = [0; 4]; /// let offset = Simd::from_array([3, 2, 1, 0]); /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); diff --git a/crates/core_simd/tests/cast.rs b/crates/core_simd/tests/cast.rs index 00545936ea2..185e1945faa 100644 --- a/crates/core_simd/tests/cast.rs +++ b/crates/core_simd/tests/cast.rs @@ -3,7 +3,7 @@ macro_rules! cast_types { ($start:ident, $($target:ident),*) => { mod $start { #[allow(unused)] - use core_simd::simd::{Simd, SimdInt, SimdUint, SimdFloat}; + use core_simd::simd::prelude::*; type Vector = Simd<$start, N>; $( mod $target { diff --git a/crates/core_simd/tests/ops_macros.rs b/crates/core_simd/tests/ops_macros.rs index 40aba2fd6cb..50faba04991 100644 --- a/crates/core_simd/tests/ops_macros.rs +++ b/crates/core_simd/tests/ops_macros.rs @@ -254,7 +254,7 @@ macro_rules! impl_common_integer_tests { macro_rules! impl_signed_tests { { $scalar:tt } => { mod $scalar { - use core_simd::simd::SimdInt; + use core_simd::simd::num::SimdInt; type Vector = core_simd::simd::Simd; type Scalar = $scalar; @@ -306,7 +306,7 @@ macro_rules! impl_signed_tests { } fn simd_min() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); assert_eq!(a.simd_min(b), a); @@ -316,7 +316,7 @@ macro_rules! impl_signed_tests { } fn simd_max() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let a = Vector::::splat(Scalar::MIN); let b = Vector::::splat(0); assert_eq!(a.simd_max(b), b); @@ -326,7 +326,7 @@ macro_rules! impl_signed_tests { } fn simd_clamp() { - use core_simd::simd::SimdOrd; + use core_simd::simd::cmp::SimdOrd; let min = Vector::::splat(Scalar::MIN); let max = Vector::::splat(Scalar::MAX); let zero = Vector::::splat(0); @@ -395,7 +395,7 @@ macro_rules! impl_signed_tests { macro_rules! impl_unsigned_tests { { $scalar:tt } => { mod $scalar { - use core_simd::simd::SimdUint; + use core_simd::simd::num::SimdUint; type Vector = core_simd::simd::Simd; type Scalar = $scalar; @@ -440,7 +440,7 @@ macro_rules! impl_unsigned_tests { macro_rules! impl_float_tests { { $scalar:tt, $int_scalar:tt } => { mod $scalar { - use core_simd::simd::SimdFloat; + use core_simd::simd::num::SimdFloat; type Vector = core_simd::simd::Simd; type Scalar = $scalar; diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index 0ae8f83b8b9..a90ff928ced 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -1,6 +1,9 @@ #![feature(portable_simd, strict_provenance)] -use core_simd::simd::{Simd, SimdConstPtr, SimdMutPtr}; +use core_simd::simd::{ + ptr::{SimdConstPtr, SimdMutPtr}, + Simd, +}; macro_rules! common_tests { { $constness:ident } => { diff --git a/crates/core_simd/tests/round.rs b/crates/core_simd/tests/round.rs index 191c39e2370..847766ec41e 100644 --- a/crates/core_simd/tests/round.rs +++ b/crates/core_simd/tests/round.rs @@ -53,7 +53,7 @@ macro_rules! float_rounding_test { test_helpers::test_lanes! { fn to_int_unchecked() { - use core_simd::simd::SimdFloat; + use core_simd::simd::num::SimdFloat; // The maximum integer that can be represented by the equivalently sized float has // all of the mantissa digits set to 1, pushed up to the MSB. const ALL_MANTISSA_BITS: IntScalar = ((1 << ::MANTISSA_DIGITS) - 1); diff --git a/crates/std_float/src/lib.rs b/crates/std_float/src/lib.rs index bb2b5a2dbba..1fef17242ca 100644 --- a/crates/std_float/src/lib.rs +++ b/crates/std_float/src/lib.rs @@ -149,7 +149,7 @@ where #[cfg(test)] mod tests { use super::*; - use simd::*; + use simd::prelude::*; #[test] fn everything_works() { -- cgit 1.4.1-3-g733a5