From 3046dbe7deaebf2804fc822dd1828b3c794edc6f Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 4 May 2024 12:34:16 +0200 Subject: move intrinsics implementations and tests into dedicated folder and make them separate from 'shims' --- src/tools/miri/src/intrinsics/atomic.rs | 288 ++++ src/tools/miri/src/intrinsics/mod.rs | 433 ++++++ src/tools/miri/src/intrinsics/simd.rs | 798 ++++++++++ src/tools/miri/src/lib.rs | 3 +- src/tools/miri/src/shims/intrinsics/atomic.rs | 288 ---- src/tools/miri/src/shims/intrinsics/mod.rs | 433 ------ src/tools/miri/src/shims/intrinsics/simd.rs | 798 ---------- src/tools/miri/src/shims/mod.rs | 1 - .../fail/intrinsics/intrinsic_target_feature.rs | 42 + .../intrinsics/intrinsic_target_feature.stderr | 15 + .../tests/fail/shims/intrinsic_target_feature.rs | 42 - .../fail/shims/intrinsic_target_feature.stderr | 15 - src/tools/miri/tests/pass/intrinsics-integer.rs | 147 -- .../miri/tests/pass/intrinsics-x86-aes-vaes.rs | 291 ---- src/tools/miri/tests/pass/intrinsics-x86-avx.rs | 1431 ----------------- src/tools/miri/tests/pass/intrinsics-x86-avx2.rs | 1613 -------------------- src/tools/miri/tests/pass/intrinsics-x86-avx512.rs | 217 --- .../pass/intrinsics-x86-pause-without-sse2.rs | 25 - src/tools/miri/tests/pass/intrinsics-x86-sse.rs | 1107 -------------- src/tools/miri/tests/pass/intrinsics-x86-sse2.rs | 848 ---------- .../miri/tests/pass/intrinsics-x86-sse3-ssse3.rs | 395 ----- src/tools/miri/tests/pass/intrinsics-x86-sse41.rs | 548 ------- src/tools/miri/tests/pass/intrinsics-x86.rs | 99 -- src/tools/miri/tests/pass/intrinsics.rs | 60 - src/tools/miri/tests/pass/intrinsics/integer.rs | 147 ++ src/tools/miri/tests/pass/intrinsics/intrinsics.rs | 60 + .../tests/pass/intrinsics/portable-simd-ptrs.rs | 12 + .../miri/tests/pass/intrinsics/portable-simd.rs | 581 +++++++ src/tools/miri/tests/pass/intrinsics/volatile.rs | 12 + src/tools/miri/tests/pass/portable-simd-ptrs.rs | 12 - src/tools/miri/tests/pass/portable-simd.rs | 581 ------- .../pass/shims/x86/intrinsics-x86-aes-vaes.rs | 291 ++++ .../tests/pass/shims/x86/intrinsics-x86-avx.rs | 1431 +++++++++++++++++ .../tests/pass/shims/x86/intrinsics-x86-avx2.rs | 1613 ++++++++++++++++++++ .../tests/pass/shims/x86/intrinsics-x86-avx512.rs | 217 +++ .../shims/x86/intrinsics-x86-pause-without-sse2.rs | 25 + .../tests/pass/shims/x86/intrinsics-x86-sse.rs | 1107 ++++++++++++++ .../tests/pass/shims/x86/intrinsics-x86-sse2.rs | 848 ++++++++++ .../pass/shims/x86/intrinsics-x86-sse3-ssse3.rs | 395 +++++ .../tests/pass/shims/x86/intrinsics-x86-sse41.rs | 548 +++++++ .../miri/tests/pass/shims/x86/intrinsics-x86.rs | 99 ++ src/tools/miri/tests/pass/volatile.rs | 12 - 42 files changed, 8964 insertions(+), 8964 deletions(-) create mode 100644 src/tools/miri/src/intrinsics/atomic.rs create mode 100644 src/tools/miri/src/intrinsics/mod.rs create mode 100644 src/tools/miri/src/intrinsics/simd.rs delete mode 100644 src/tools/miri/src/shims/intrinsics/atomic.rs delete mode 100644 src/tools/miri/src/shims/intrinsics/mod.rs delete mode 100644 src/tools/miri/src/shims/intrinsics/simd.rs create mode 100644 src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs create mode 100644 src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr delete mode 100644 src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs delete mode 100644 src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr delete mode 100644 src/tools/miri/tests/pass/intrinsics-integer.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-avx.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-avx2.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-avx512.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-sse.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-sse2.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86-sse41.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics-x86.rs delete mode 100644 src/tools/miri/tests/pass/intrinsics.rs create mode 100644 src/tools/miri/tests/pass/intrinsics/integer.rs create mode 100644 src/tools/miri/tests/pass/intrinsics/intrinsics.rs create mode 100644 src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs create mode 100644 src/tools/miri/tests/pass/intrinsics/portable-simd.rs create mode 100644 src/tools/miri/tests/pass/intrinsics/volatile.rs delete mode 100644 src/tools/miri/tests/pass/portable-simd-ptrs.rs delete mode 100644 src/tools/miri/tests/pass/portable-simd.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs create mode 100644 src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs delete mode 100644 src/tools/miri/tests/pass/volatile.rs (limited to 'src') diff --git a/src/tools/miri/src/intrinsics/atomic.rs b/src/tools/miri/src/intrinsics/atomic.rs new file mode 100644 index 00000000000..865886a7fc1 --- /dev/null +++ b/src/tools/miri/src/intrinsics/atomic.rs @@ -0,0 +1,288 @@ +use rustc_middle::{mir, mir::BinOp, ty}; + +use crate::*; +use helpers::check_arg_count; + +pub enum AtomicOp { + /// The `bool` indicates whether the result of the operation should be negated + /// (must be a boolean-typed operation). + MirOp(mir::BinOp, bool), + Max, + Min, +} + +impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} +pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { + /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed. + fn emulate_atomic_intrinsic( + &mut self, + intrinsic_name: &str, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect(); + + fn read_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicReadOrd> { + Ok(match ord { + "seqcst" => AtomicReadOrd::SeqCst, + "acquire" => AtomicReadOrd::Acquire, + "relaxed" => AtomicReadOrd::Relaxed, + _ => throw_unsup_format!("unsupported read ordering `{ord}`"), + }) + } + + fn write_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicWriteOrd> { + Ok(match ord { + "seqcst" => AtomicWriteOrd::SeqCst, + "release" => AtomicWriteOrd::Release, + "relaxed" => AtomicWriteOrd::Relaxed, + _ => throw_unsup_format!("unsupported write ordering `{ord}`"), + }) + } + + fn rw_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicRwOrd> { + Ok(match ord { + "seqcst" => AtomicRwOrd::SeqCst, + "acqrel" => AtomicRwOrd::AcqRel, + "acquire" => AtomicRwOrd::Acquire, + "release" => AtomicRwOrd::Release, + "relaxed" => AtomicRwOrd::Relaxed, + _ => throw_unsup_format!("unsupported read-write ordering `{ord}`"), + }) + } + + fn fence_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicFenceOrd> { + Ok(match ord { + "seqcst" => AtomicFenceOrd::SeqCst, + "acqrel" => AtomicFenceOrd::AcqRel, + "acquire" => AtomicFenceOrd::Acquire, + "release" => AtomicFenceOrd::Release, + _ => throw_unsup_format!("unsupported fence ordering `{ord}`"), + }) + } + + match &*intrinsic_structure { + ["load", ord] => this.atomic_load(args, dest, read_ord(ord)?)?, + ["store", ord] => this.atomic_store(args, write_ord(ord)?)?, + + ["fence", ord] => this.atomic_fence_intrinsic(args, fence_ord(ord)?)?, + ["singlethreadfence", ord] => this.compiler_fence_intrinsic(args, fence_ord(ord)?)?, + + ["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord)?)?, + ["cxchg", ord1, ord2] => + this.atomic_compare_exchange(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?, + ["cxchgweak", ord1, ord2] => + this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?, + + ["or", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord)?)?, + ["xor", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord)?)?, + ["and", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord)?)?, + ["nand", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord)?)?, + ["xadd", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord)?)?, + ["xsub", ord] => + this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord)?)?, + ["min", ord] => { + // Later we will use the type to indicate signed vs unsigned, + // so make sure it matches the intrinsic name. + assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); + this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?; + } + ["umin", ord] => { + // Later we will use the type to indicate signed vs unsigned, + // so make sure it matches the intrinsic name. + assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); + this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?; + } + ["max", ord] => { + // Later we will use the type to indicate signed vs unsigned, + // so make sure it matches the intrinsic name. + assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); + this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?; + } + ["umax", ord] => { + // Later we will use the type to indicate signed vs unsigned, + // so make sure it matches the intrinsic name. + assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); + this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?; + } + + _ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"), + } + Ok(()) + } +} + +impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for MiriInterpCx<'mir, 'tcx> {} +trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { + fn atomic_load( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + atomic: AtomicReadOrd, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let [place] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + + // Perform atomic load. + let val = this.read_scalar_atomic(&place, atomic)?; + // Perform regular store. + this.write_scalar(val, dest)?; + Ok(()) + } + + fn atomic_store( + &mut self, + args: &[OpTy<'tcx, Provenance>], + atomic: AtomicWriteOrd, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let [place, val] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + + // Perform regular load. + let val = this.read_scalar(val)?; + // Perform atomic store + this.write_scalar_atomic(val, &place, atomic)?; + Ok(()) + } + + fn compiler_fence_intrinsic( + &mut self, + args: &[OpTy<'tcx, Provenance>], + atomic: AtomicFenceOrd, + ) -> InterpResult<'tcx> { + let [] = check_arg_count(args)?; + let _ = atomic; + //FIXME: compiler fences are currently ignored + Ok(()) + } + + fn atomic_fence_intrinsic( + &mut self, + args: &[OpTy<'tcx, Provenance>], + atomic: AtomicFenceOrd, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + let [] = check_arg_count(args)?; + this.atomic_fence(atomic)?; + Ok(()) + } + + fn atomic_rmw_op( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + atomic_op: AtomicOp, + atomic: AtomicRwOrd, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let [place, rhs] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + let rhs = this.read_immediate(rhs)?; + + if !place.layout.ty.is_integral() && !place.layout.ty.is_unsafe_ptr() { + span_bug!( + this.cur_span(), + "atomic arithmetic operations only work on integer and raw pointer types", + ); + } + if rhs.layout.ty != place.layout.ty { + span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch"); + } + + match atomic_op { + AtomicOp::Min => { + let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?; + this.write_immediate(*old, dest)?; // old value is returned + Ok(()) + } + AtomicOp::Max => { + let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?; + this.write_immediate(*old, dest)?; // old value is returned + Ok(()) + } + AtomicOp::MirOp(op, neg) => { + let old = this.atomic_rmw_op_immediate(&place, &rhs, op, neg, atomic)?; + this.write_immediate(*old, dest)?; // old value is returned + Ok(()) + } + } + } + + fn atomic_exchange( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + atomic: AtomicRwOrd, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let [place, new] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + let new = this.read_scalar(new)?; + + let old = this.atomic_exchange_scalar(&place, new, atomic)?; + this.write_scalar(old, dest)?; // old value is returned + Ok(()) + } + + fn atomic_compare_exchange_impl( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + success: AtomicRwOrd, + fail: AtomicReadOrd, + can_fail_spuriously: bool, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let [place, expect_old, new] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()` + let new = this.read_scalar(new)?; + + let old = this.atomic_compare_exchange_scalar( + &place, + &expect_old, + new, + success, + fail, + can_fail_spuriously, + )?; + + // Return old value. + this.write_immediate(old, dest)?; + Ok(()) + } + + fn atomic_compare_exchange( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + success: AtomicRwOrd, + fail: AtomicReadOrd, + ) -> InterpResult<'tcx> { + self.atomic_compare_exchange_impl(args, dest, success, fail, false) + } + + fn atomic_compare_exchange_weak( + &mut self, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + success: AtomicRwOrd, + fail: AtomicReadOrd, + ) -> InterpResult<'tcx> { + self.atomic_compare_exchange_impl(args, dest, success, fail, true) + } +} diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs new file mode 100644 index 00000000000..a7ba4fd7f9e --- /dev/null +++ b/src/tools/miri/src/intrinsics/mod.rs @@ -0,0 +1,433 @@ +mod atomic; +mod simd; + +use std::iter; + +use rand::Rng; +use rustc_apfloat::{Float, Round}; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::{ + mir, + ty::{self, FloatTy}, +}; +use rustc_target::abi::Size; + +use crate::*; +use atomic::EvalContextExt as _; +use helpers::{check_arg_count, ToHost, ToSoft}; +use simd::EvalContextExt as _; + +impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} +pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { + fn call_intrinsic( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + ret: Option, + _unwind: mir::UnwindAction, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + // See if the core engine can handle this intrinsic. + if this.emulate_intrinsic(instance, args, dest, ret)? { + return Ok(()); + } + let intrinsic_name = this.tcx.item_name(instance.def_id()); + let intrinsic_name = intrinsic_name.as_str(); + + // Handle intrinsics without return place. + match intrinsic_name { + "abort" => { + throw_machine_stop!(TerminationInfo::Abort( + "the program aborted execution".to_owned() + )) + } + _ => {} + } + + // All remaining supported intrinsics have a return place. + let ret = match ret { + None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"), + Some(p) => p, + }; + + // Some intrinsics are special and need the "ret". + match intrinsic_name { + "catch_unwind" => return this.handle_catch_unwind(args, dest, ret), + _ => {} + } + + // The rest jumps to `ret` immediately. + this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest)?; + + trace!("{:?}", this.dump_place(&dest.clone().into())); + this.go_to_block(ret); + Ok(()) + } + + /// Emulates a Miri-supported intrinsic (not supported by the core engine). + fn emulate_intrinsic_by_name( + &mut self, + intrinsic_name: &str, + generic_args: ty::GenericArgsRef<'tcx>, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + if let Some(name) = intrinsic_name.strip_prefix("atomic_") { + return this.emulate_atomic_intrinsic(name, args, dest); + } + if let Some(name) = intrinsic_name.strip_prefix("simd_") { + return this.emulate_simd_intrinsic(name, generic_args, args, dest); + } + + match intrinsic_name { + // Miri overwriting CTFE intrinsics. + "ptr_guaranteed_cmp" => { + let [left, right] = check_arg_count(args)?; + let left = this.read_immediate(left)?; + let right = this.read_immediate(right)?; + let val = this.wrapping_binary_op(mir::BinOp::Eq, &left, &right)?; + // We're type punning a bool as an u8 here. + this.write_scalar(val.to_scalar(), dest)?; + } + "const_allocate" => { + // For now, for compatibility with the run-time implementation of this, we just return null. + // See . + this.write_null(dest)?; + } + "const_deallocate" => { + // complete NOP + } + + // Raw memory accesses + "volatile_load" => { + let [place] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + this.copy_op(&place, dest)?; + } + "volatile_store" => { + let [place, dest] = check_arg_count(args)?; + let place = this.deref_pointer(place)?; + this.copy_op(dest, &place)?; + } + + "write_bytes" | "volatile_set_memory" => { + let [ptr, val_byte, count] = check_arg_count(args)?; + let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty; + let ty_layout = this.layout_of(ty)?; + let val_byte = this.read_scalar(val_byte)?.to_u8()?; + let ptr = this.read_pointer(ptr)?; + let count = this.read_target_usize(count)?; + // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max), + // but no actual allocation can be big enough for the difference to be noticeable. + let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| { + err_ub_format!("overflow computing total size of `{intrinsic_name}`") + })?; + this.write_bytes_ptr(ptr, iter::repeat(val_byte).take(byte_count.bytes_usize()))?; + } + + // Memory model / provenance manipulation + "ptr_mask" => { + let [ptr, mask] = check_arg_count(args)?; + + let ptr = this.read_pointer(ptr)?; + let mask = this.read_target_usize(mask)?; + + let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask); + + this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?; + } + + // We want to return either `true` or `false` at random, or else something like + // ``` + // if !is_val_statically_known(0) { unreachable_unchecked(); } + // ``` + // Would not be considered UB, or the other way around (`is_val_statically_known(0)`). + "is_val_statically_known" => { + let [arg] = check_arg_count(args)?; + this.validate_operand(arg)?; + let branch: bool = this.machine.rng.get_mut().gen(); + this.write_scalar(Scalar::from_bool(branch), dest)?; + } + + // Floating-point operations + "fabsf32" => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f32()?; + // This is a "bitwise" operation, so there's no NaN non-determinism. + this.write_scalar(Scalar::from_f32(f.abs()), dest)?; + } + "fabsf64" => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f64()?; + // This is a "bitwise" operation, so there's no NaN non-determinism. + this.write_scalar(Scalar::from_f64(f.abs()), dest)?; + } + "floorf32" | "ceilf32" | "truncf32" | "roundf32" | "rintf32" => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f32()?; + let mode = match intrinsic_name { + "floorf32" => Round::TowardNegative, + "ceilf32" => Round::TowardPositive, + "truncf32" => Round::TowardZero, + "roundf32" => Round::NearestTiesToAway, + "rintf32" => Round::NearestTiesToEven, + _ => bug!(), + }; + let res = f.round_to_integral(mode).value; + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + #[rustfmt::skip] + | "sinf32" + | "cosf32" + | "sqrtf32" + | "expf32" + | "exp2f32" + | "logf32" + | "log10f32" + | "log2f32" + => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f32()?; + // Using host floats (but it's fine, these operations do not have guaranteed precision). + let f_host = f.to_host(); + let res = match intrinsic_name { + "sinf32" => f_host.sin(), + "cosf32" => f_host.cos(), + "sqrtf32" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats + "expf32" => f_host.exp(), + "exp2f32" => f_host.exp2(), + "logf32" => f_host.ln(), + "log10f32" => f_host.log10(), + "log2f32" => f_host.log2(), + _ => bug!(), + }; + let res = res.to_soft(); + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + + "floorf64" | "ceilf64" | "truncf64" | "roundf64" | "rintf64" => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f64()?; + let mode = match intrinsic_name { + "floorf64" => Round::TowardNegative, + "ceilf64" => Round::TowardPositive, + "truncf64" => Round::TowardZero, + "roundf64" => Round::NearestTiesToAway, + "rintf64" => Round::NearestTiesToEven, + _ => bug!(), + }; + let res = f.round_to_integral(mode).value; + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + #[rustfmt::skip] + | "sinf64" + | "cosf64" + | "sqrtf64" + | "expf64" + | "exp2f64" + | "logf64" + | "log10f64" + | "log2f64" + => { + let [f] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f64()?; + // Using host floats (but it's fine, these operations do not have guaranteed precision). + let f_host = f.to_host(); + let res = match intrinsic_name { + "sinf64" => f_host.sin(), + "cosf64" => f_host.cos(), + "sqrtf64" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats + "expf64" => f_host.exp(), + "exp2f64" => f_host.exp2(), + "logf64" => f_host.ln(), + "log10f64" => f_host.log10(), + "log2f64" => f_host.log2(), + _ => bug!(), + }; + let res = res.to_soft(); + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + + #[rustfmt::skip] + | "fadd_fast" + | "fsub_fast" + | "fmul_fast" + | "fdiv_fast" + | "frem_fast" + => { + let [a, b] = check_arg_count(args)?; + let a = this.read_immediate(a)?; + let b = this.read_immediate(b)?; + let op = match intrinsic_name { + "fadd_fast" => mir::BinOp::Add, + "fsub_fast" => mir::BinOp::Sub, + "fmul_fast" => mir::BinOp::Mul, + "fdiv_fast" => mir::BinOp::Div, + "frem_fast" => mir::BinOp::Rem, + _ => bug!(), + }; + let float_finite = |x: &ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> { + let ty::Float(fty) = x.layout.ty.kind() else { + bug!("float_finite: non-float input type {}", x.layout.ty) + }; + Ok(match fty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(), + FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(), + FloatTy::F128 => unimplemented!("f16_f128"), + }) + }; + match (float_finite(&a)?, float_finite(&b)?) { + (false, false) => throw_ub_format!( + "`{intrinsic_name}` intrinsic called with non-finite value as both parameters", + ), + (false, _) => throw_ub_format!( + "`{intrinsic_name}` intrinsic called with non-finite value as first parameter", + ), + (_, false) => throw_ub_format!( + "`{intrinsic_name}` intrinsic called with non-finite value as second parameter", + ), + _ => {} + } + let res = this.wrapping_binary_op(op, &a, &b)?; + if !float_finite(&res)? { + throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result"); + } + // This cannot be a NaN so we also don't have to apply any non-determinism. + // (Also, `wrapping_binary_op` already called `generate_nan` if needed.) + this.write_immediate(*res, dest)?; + } + + #[rustfmt::skip] + | "minnumf32" + | "maxnumf32" + | "copysignf32" + => { + let [a, b] = check_arg_count(args)?; + let a = this.read_scalar(a)?.to_f32()?; + let b = this.read_scalar(b)?.to_f32()?; + let res = match intrinsic_name { + "minnumf32" => this.adjust_nan(a.min(b), &[a, b]), + "maxnumf32" => this.adjust_nan(a.max(b), &[a, b]), + "copysignf32" => a.copy_sign(b), // bitwise, no NaN adjustments + _ => bug!(), + }; + this.write_scalar(Scalar::from_f32(res), dest)?; + } + + #[rustfmt::skip] + | "minnumf64" + | "maxnumf64" + | "copysignf64" + => { + let [a, b] = check_arg_count(args)?; + let a = this.read_scalar(a)?.to_f64()?; + let b = this.read_scalar(b)?.to_f64()?; + let res = match intrinsic_name { + "minnumf64" => this.adjust_nan(a.min(b), &[a, b]), + "maxnumf64" => this.adjust_nan(a.max(b), &[a, b]), + "copysignf64" => a.copy_sign(b), // bitwise, no NaN adjustments + _ => bug!(), + }; + this.write_scalar(Scalar::from_f64(res), dest)?; + } + + "fmaf32" => { + let [a, b, c] = check_arg_count(args)?; + let a = this.read_scalar(a)?.to_f32()?; + let b = this.read_scalar(b)?.to_f32()?; + let c = this.read_scalar(c)?.to_f32()?; + // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11 + let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); + let res = this.adjust_nan(res, &[a, b, c]); + this.write_scalar(res, dest)?; + } + + "fmaf64" => { + let [a, b, c] = check_arg_count(args)?; + let a = this.read_scalar(a)?.to_f64()?; + let b = this.read_scalar(b)?.to_f64()?; + let c = this.read_scalar(c)?.to_f64()?; + // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11 + let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); + let res = this.adjust_nan(res, &[a, b, c]); + this.write_scalar(res, dest)?; + } + + "powf32" => { + let [f1, f2] = check_arg_count(args)?; + let f1 = this.read_scalar(f1)?.to_f32()?; + let f2 = this.read_scalar(f2)?.to_f32()?; + // Using host floats (but it's fine, this operation does not have guaranteed precision). + let res = f1.to_host().powf(f2.to_host()).to_soft(); + let res = this.adjust_nan(res, &[f1, f2]); + this.write_scalar(res, dest)?; + } + + "powf64" => { + let [f1, f2] = check_arg_count(args)?; + let f1 = this.read_scalar(f1)?.to_f64()?; + let f2 = this.read_scalar(f2)?.to_f64()?; + // Using host floats (but it's fine, this operation does not have guaranteed precision). + let res = f1.to_host().powf(f2.to_host()).to_soft(); + let res = this.adjust_nan(res, &[f1, f2]); + this.write_scalar(res, dest)?; + } + + "powif32" => { + let [f, i] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f32()?; + let i = this.read_scalar(i)?.to_i32()?; + // Using host floats (but it's fine, this operation does not have guaranteed precision). + let res = f.to_host().powi(i).to_soft(); + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + + "powif64" => { + let [f, i] = check_arg_count(args)?; + let f = this.read_scalar(f)?.to_f64()?; + let i = this.read_scalar(i)?.to_i32()?; + // Using host floats (but it's fine, this operation does not have guaranteed precision). + let res = f.to_host().powi(i).to_soft(); + let res = this.adjust_nan(res, &[f]); + this.write_scalar(res, dest)?; + } + + "float_to_int_unchecked" => { + let [val] = check_arg_count(args)?; + let val = this.read_immediate(val)?; + + let res = this + .float_to_int_checked(&val, dest.layout, Round::TowardZero)? + .ok_or_else(|| { + err_ub_format!( + "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`", + dest.layout.ty + ) + })?; + + this.write_immediate(*res, dest)?; + } + + // Other + "breakpoint" => { + let [] = check_arg_count(args)?; + // normally this would raise a SIGTRAP, which aborts if no debugger is connected + throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap"))) + } + + name => throw_unsup_format!("unimplemented intrinsic: `{name}`"), + } + + Ok(()) + } +} diff --git a/src/tools/miri/src/intrinsics/simd.rs b/src/tools/miri/src/intrinsics/simd.rs new file mode 100644 index 00000000000..e6d6f72404e --- /dev/null +++ b/src/tools/miri/src/intrinsics/simd.rs @@ -0,0 +1,798 @@ +use rustc_apfloat::{Float, Round}; +use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::{mir, ty, ty::FloatTy}; +use rustc_span::{sym, Symbol}; +use rustc_target::abi::{Endian, HasDataLayout}; + +use crate::helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool, ToHost, ToSoft}; +use crate::*; + +#[derive(Copy, Clone)] +pub(crate) enum MinMax { + Min, + Max, +} + +impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} +pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { + /// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed. + fn emulate_simd_intrinsic( + &mut self, + intrinsic_name: &str, + generic_args: ty::GenericArgsRef<'tcx>, + args: &[OpTy<'tcx, Provenance>], + dest: &MPlaceTy<'tcx, Provenance>, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + match intrinsic_name { + #[rustfmt::skip] + | "neg" + | "fabs" + | "ceil" + | "floor" + | "round" + | "trunc" + | "fsqrt" + | "fsin" + | "fcos" + | "fexp" + | "fexp2" + | "flog" + | "flog2" + | "flog10" + | "ctlz" + | "cttz" + | "bswap" + | "bitreverse" + => { + let [op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, op_len); + + #[derive(Copy, Clone)] + enum Op<'a> { + MirOp(mir::UnOp), + Abs, + Round(rustc_apfloat::Round), + Numeric(Symbol), + HostOp(&'a str), + } + let which = match intrinsic_name { + "neg" => Op::MirOp(mir::UnOp::Neg), + "fabs" => Op::Abs, + "ceil" => Op::Round(rustc_apfloat::Round::TowardPositive), + "floor" => Op::Round(rustc_apfloat::Round::TowardNegative), + "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway), + "trunc" => Op::Round(rustc_apfloat::Round::TowardZero), + "ctlz" => Op::Numeric(sym::ctlz), + "cttz" => Op::Numeric(sym::cttz), + "bswap" => Op::Numeric(sym::bswap), + "bitreverse" => Op::Numeric(sym::bitreverse), + _ => Op::HostOp(intrinsic_name), + }; + + for i in 0..dest_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + let dest = this.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // This already does NaN adjustments + this.wrapping_unary_op(mir_op, &op)?.to_scalar() + } + Op::Abs => { + // Works for f32 and f64. + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + }; + let op = op.to_scalar(); + // "Bitwise" operation, no NaN adjustments + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()), + FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()), + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::HostOp(host_op) => { + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + }; + // Using host floats (but it's fine, these operations do not have guaranteed precision). + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let f = op.to_scalar().to_f32()?; + let f_host = f.to_host(); + let res = match host_op { + "fsqrt" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats + "fsin" => f_host.sin(), + "fcos" => f_host.cos(), + "fexp" => f_host.exp(), + "fexp2" => f_host.exp2(), + "flog" => f_host.ln(), + "flog2" => f_host.log2(), + "flog10" => f_host.log10(), + _ => bug!(), + }; + let res = res.to_soft(); + let res = this.adjust_nan(res, &[f]); + Scalar::from(res) + } + FloatTy::F64 => { + let f = op.to_scalar().to_f64()?; + let f_host = f.to_host(); + let res = match host_op { + "fsqrt" => f_host.sqrt(), + "fsin" => f_host.sin(), + "fcos" => f_host.cos(), + "fexp" => f_host.exp(), + "fexp2" => f_host.exp2(), + "flog" => f_host.ln(), + "flog2" => f_host.log2(), + "flog10" => f_host.log10(), + _ => bug!(), + }; + let res = res.to_soft(); + let res = this.adjust_nan(res, &[f]); + Scalar::from(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Round(rounding) => { + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + }; + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let f = op.to_scalar().to_f32()?; + let res = f.round_to_integral(rounding).value; + let res = this.adjust_nan(res, &[f]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let f = op.to_scalar().to_f64()?; + let res = f.round_to_integral(rounding).value; + let res = this.adjust_nan(res, &[f]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Numeric(name) => { + this.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? + } + }; + this.write_scalar(val, &dest)?; + } + } + #[rustfmt::skip] + | "add" + | "sub" + | "mul" + | "div" + | "rem" + | "shl" + | "shr" + | "and" + | "or" + | "xor" + | "eq" + | "ne" + | "lt" + | "le" + | "gt" + | "ge" + | "fmax" + | "fmin" + | "saturating_add" + | "saturating_sub" + | "arith_offset" + => { + use mir::BinOp; + + let [left, right] = check_arg_count(args)?; + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, left_len); + assert_eq!(dest_len, right_len); + + enum Op { + MirOp(BinOp), + SaturatingOp(BinOp), + FMinMax(MinMax), + WrappingOffset, + } + let which = match intrinsic_name { + "add" => Op::MirOp(BinOp::Add), + "sub" => Op::MirOp(BinOp::Sub), + "mul" => Op::MirOp(BinOp::Mul), + "div" => Op::MirOp(BinOp::Div), + "rem" => Op::MirOp(BinOp::Rem), + "shl" => Op::MirOp(BinOp::Shl), + "shr" => Op::MirOp(BinOp::Shr), + "and" => Op::MirOp(BinOp::BitAnd), + "or" => Op::MirOp(BinOp::BitOr), + "xor" => Op::MirOp(BinOp::BitXor), + "eq" => Op::MirOp(BinOp::Eq), + "ne" => Op::MirOp(BinOp::Ne), + "lt" => Op::MirOp(BinOp::Lt), + "le" => Op::MirOp(BinOp::Le), + "gt" => Op::MirOp(BinOp::Gt), + "ge" => Op::MirOp(BinOp::Ge), + "fmax" => Op::FMinMax(MinMax::Max), + "fmin" => Op::FMinMax(MinMax::Min), + "saturating_add" => Op::SaturatingOp(BinOp::Add), + "saturating_sub" => Op::SaturatingOp(BinOp::Sub), + "arith_offset" => Op::WrappingOffset, + _ => unreachable!(), + }; + + for i in 0..dest_len { + let left = this.read_immediate(&this.project_index(&left, i)?)?; + let right = this.read_immediate(&this.project_index(&right, i)?)?; + let dest = this.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // This does NaN adjustments. + let (val, overflowed) = this.overflowing_binary_op(mir_op, &left, &right)?; + if matches!(mir_op, BinOp::Shl | BinOp::Shr) { + // Shifts have extra UB as SIMD operations that the MIR binop does not have. + // See . + if overflowed { + let r_val = right.to_scalar().to_bits(right.layout.size)?; + throw_ub_format!("overflowing shift by {r_val} in `simd_{intrinsic_name}` in SIMD lane {i}"); + } + } + if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) { + // Special handling for boolean-returning operations + assert_eq!(val.layout.ty, this.tcx.types.bool); + let val = val.to_scalar().to_bool().unwrap(); + bool_to_simd_element(val, dest.layout.size) + } else { + assert_ne!(val.layout.ty, this.tcx.types.bool); + assert_eq!(val.layout.ty, dest.layout.ty); + val.to_scalar() + } + } + Op::SaturatingOp(mir_op) => { + this.saturating_arith(mir_op, &left, &right)? + } + Op::WrappingOffset => { + let ptr = left.to_scalar().to_pointer(this)?; + let offset_count = right.to_scalar().to_target_isize(this)?; + let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty; + + let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap(); + let offset_bytes = offset_count.wrapping_mul(pointee_size); + let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this); + Scalar::from_maybe_pointer(offset_ptr, this) + } + Op::FMinMax(op) => { + this.fminmax_op(op, &left, &right)? + } + }; + this.write_scalar(val, &dest)?; + } + } + "fma" => { + let [a, b, c] = check_arg_count(args)?; + let (a, a_len) = this.operand_to_simd(a)?; + let (b, b_len) = this.operand_to_simd(b)?; + let (c, c_len) = this.operand_to_simd(c)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, a_len); + assert_eq!(dest_len, b_len); + assert_eq!(dest_len, c_len); + + for i in 0..dest_len { + let a = this.read_scalar(&this.project_index(&a, i)?)?; + let b = this.read_scalar(&this.project_index(&b, i)?)?; + let c = this.read_scalar(&this.project_index(&c, i)?)?; + let dest = this.project_index(&dest, i)?; + + // Works for f32 and f64. + // FIXME: using host floats to work around https://github.com/rust-lang/miri/issues/2468. + let ty::Float(float_ty) = dest.layout.ty.kind() else { + span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) + }; + let val = match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let a = a.to_f32()?; + let b = b.to_f32()?; + let c = c.to_f32()?; + let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); + let res = this.adjust_nan(res, &[a, b, c]); + Scalar::from(res) + } + FloatTy::F64 => { + let a = a.to_f64()?; + let b = b.to_f64()?; + let c = c.to_f64()?; + let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); + let res = this.adjust_nan(res, &[a, b, c]); + Scalar::from(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + }; + this.write_scalar(val, &dest)?; + } + } + #[rustfmt::skip] + | "reduce_and" + | "reduce_or" + | "reduce_xor" + | "reduce_any" + | "reduce_all" + | "reduce_max" + | "reduce_min" => { + use mir::BinOp; + + let [op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + + let imm_from_bool = + |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool); + + enum Op { + MirOp(BinOp), + MirOpBool(BinOp), + MinMax(MinMax), + } + let which = match intrinsic_name { + "reduce_and" => Op::MirOp(BinOp::BitAnd), + "reduce_or" => Op::MirOp(BinOp::BitOr), + "reduce_xor" => Op::MirOp(BinOp::BitXor), + "reduce_any" => Op::MirOpBool(BinOp::BitOr), + "reduce_all" => Op::MirOpBool(BinOp::BitAnd), + "reduce_max" => Op::MinMax(MinMax::Max), + "reduce_min" => Op::MinMax(MinMax::Min), + _ => unreachable!(), + }; + + // Initialize with first lane, then proceed with the rest. + let mut res = this.read_immediate(&this.project_index(&op, 0)?)?; + if matches!(which, Op::MirOpBool(_)) { + // Convert to `bool` scalar. + res = imm_from_bool(simd_element_to_bool(res)?); + } + for i in 1..op_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + res = match which { + Op::MirOp(mir_op) => { + this.wrapping_binary_op(mir_op, &res, &op)? + } + Op::MirOpBool(mir_op) => { + let op = imm_from_bool(simd_element_to_bool(op)?); + this.wrapping_binary_op(mir_op, &res, &op)? + } + Op::MinMax(mmop) => { + if matches!(res.layout.ty.kind(), ty::Float(_)) { + ImmTy::from_scalar(this.fminmax_op(mmop, &res, &op)?, res.layout) + } else { + // Just boring integers, so NaNs to worry about + let mirop = match mmop { + MinMax::Min => BinOp::Le, + MinMax::Max => BinOp::Ge, + }; + if this.wrapping_binary_op(mirop, &res, &op)?.to_scalar().to_bool()? { + res + } else { + op + } + } + } + }; + } + this.write_immediate(*res, dest)?; + } + #[rustfmt::skip] + | "reduce_add_ordered" + | "reduce_mul_ordered" => { + use mir::BinOp; + + let [op, init] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + let init = this.read_immediate(init)?; + + let mir_op = match intrinsic_name { + "reduce_add_ordered" => BinOp::Add, + "reduce_mul_ordered" => BinOp::Mul, + _ => unreachable!(), + }; + + let mut res = init; + for i in 0..op_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + res = this.wrapping_binary_op(mir_op, &res, &op)?; + } + this.write_immediate(*res, dest)?; + } + "select" => { + let [mask, yes, no] = check_arg_count(args)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let (yes, yes_len) = this.operand_to_simd(yes)?; + let (no, no_len) = this.operand_to_simd(no)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + for i in 0..dest_len { + let mask = this.read_immediate(&this.project_index(&mask, i)?)?; + let yes = this.read_immediate(&this.project_index(&yes, i)?)?; + let no = this.read_immediate(&this.project_index(&no, i)?)?; + let dest = this.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { yes } else { no }; + this.write_immediate(*val, &dest)?; + } + } + // Variant of `select` that takes a bitmask rather than a "vector of bool". + "select_bitmask" => { + let [mask, yes, no] = check_arg_count(args)?; + let (yes, yes_len) = this.operand_to_simd(yes)?; + let (no, no_len) = this.operand_to_simd(no)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + let bitmask_len = dest_len.next_multiple_of(8); + + // The mask must be an integer or an array. + assert!( + mask.layout.ty.is_integral() + || matches!(mask.layout.ty.kind(), ty::Array(elemty, _) if elemty == &this.tcx.types.u8) + ); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, mask.layout.size.bits()); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + let dest_len = u32::try_from(dest_len).unwrap(); + let bitmask_len = u32::try_from(bitmask_len).unwrap(); + + // To read the mask, we transmute it to an integer. + // That does the right thing wrt endianness. + let mask_ty = this.machine.layouts.uint(mask.layout.size).unwrap(); + let mask = mask.transmute(mask_ty, this)?; + let mask: u64 = this.read_scalar(&mask)?.to_bits(mask_ty.size)?.try_into().unwrap(); + + for i in 0..dest_len { + let bit_i = simd_bitmask_index(i, dest_len, this.data_layout().endian); + let mask = mask & 1u64.checked_shl(bit_i).unwrap(); + let yes = this.read_immediate(&this.project_index(&yes, i.into())?)?; + let no = this.read_immediate(&this.project_index(&no, i.into())?)?; + let dest = this.project_index(&dest, i.into())?; + + let val = if mask != 0 { yes } else { no }; + this.write_immediate(*val, &dest)?; + } + for i in dest_len..bitmask_len { + // If the mask is "padded", ensure that padding is all-zero. + // This deliberately does not use `simd_bitmask_index`; these bits are outside + // the bitmask. It does not matter in which order we check them. + let mask = mask & 1u64.checked_shl(i).unwrap(); + if mask != 0 { + throw_ub_format!( + "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits" + ); + } + } + } + // Converts a "vector of bool" into a bitmask. + "bitmask" => { + let [op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + let bitmask_len = op_len.next_multiple_of(8); + + // Returns either an unsigned integer or array of `u8`. + assert!( + dest.layout.ty.is_integral() + || matches!(dest.layout.ty.kind(), ty::Array(elemty, _) if elemty == &this.tcx.types.u8) + ); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, dest.layout.size.bits()); + let op_len = u32::try_from(op_len).unwrap(); + + let mut res = 0u64; + for i in 0..op_len { + let op = this.read_immediate(&this.project_index(&op, i.into())?)?; + if simd_element_to_bool(op)? { + res |= 1u64 + .checked_shl(simd_bitmask_index(i, op_len, this.data_layout().endian)) + .unwrap(); + } + } + // We have to change the type of the place to be able to write `res` into it. This + // transmutes the integer to an array, which does the right thing wrt endianness. + let dest = + dest.transmute(this.machine.layouts.uint(dest.layout.size).unwrap(), this)?; + this.write_int(res, &dest)?; + } + "cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => { + let [op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, op_len); + + let unsafe_cast = intrinsic_name == "cast"; + let safe_cast = intrinsic_name == "as"; + let ptr_cast = intrinsic_name == "cast_ptr"; + let expose_cast = intrinsic_name == "expose_provenance"; + let from_exposed_cast = intrinsic_name == "with_exposed_provenance"; + + for i in 0..dest_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + let dest = this.project_index(&dest, i)?; + + let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { + // Int-to-(int|float): always safe + (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) + if safe_cast || unsafe_cast => + this.int_to_int_or_float(&op, dest.layout)?, + // Float-to-float: always safe + (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast => + this.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in safe mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast => + this.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in unchecked mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => { + this.float_to_int_checked(&op, dest.layout, Round::TowardZero)? + .ok_or_else(|| { + err_ub_format!( + "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`", + dest.layout.ty + ) + })? + } + // Ptr-to-ptr cast + (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast => + this.ptr_to_ptr(&op, dest.layout)?, + // Ptr/Int casts + (ty::RawPtr(..), ty::Int(_) | ty::Uint(_)) if expose_cast => + this.pointer_expose_provenance_cast(&op, dest.layout)?, + (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast => + this.pointer_with_exposed_provenance_cast(&op, dest.layout)?, + // Error otherwise + _ => + throw_unsup_format!( + "Unsupported SIMD cast from element type {from_ty} to {to_ty}", + from_ty = op.layout.ty, + to_ty = dest.layout.ty, + ), + }; + this.write_immediate(*val, &dest)?; + } + } + "shuffle_generic" => { + let [left, right] = check_arg_count(args)?; + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + let index = generic_args[2] + .expect_const() + .eval(*this.tcx, this.param_env(), this.tcx.span) + .unwrap() + .unwrap_branch(); + let index_len = index.len(); + + assert_eq!(left_len, right_len); + assert_eq!(index_len as u64, dest_len); + + for i in 0..dest_len { + let src_index: u64 = index[usize::try_from(i).unwrap()] + .unwrap_leaf() + .try_to_u32() + .unwrap() + .into(); + let dest = this.project_index(&dest, i)?; + + let val = if src_index < left_len { + this.read_immediate(&this.project_index(&left, src_index)?)? + } else if src_index < left_len.checked_add(right_len).unwrap() { + let right_idx = src_index.checked_sub(left_len).unwrap(); + this.read_immediate(&this.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + this.write_immediate(*val, &dest)?; + } + } + "shuffle" => { + let [left, right, index] = check_arg_count(args)?; + let (left, left_len) = this.operand_to_simd(left)?; + let (right, right_len) = this.operand_to_simd(right)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + // `index` is an array, not a SIMD type + let ty::Array(_, index_len) = index.layout.ty.kind() else { + span_bug!( + this.cur_span(), + "simd_shuffle index argument has non-array type {}", + index.layout.ty + ) + }; + let index_len = index_len.eval_target_usize(*this.tcx, this.param_env()); + + assert_eq!(left_len, right_len); + assert_eq!(index_len, dest_len); + + for i in 0..dest_len { + let src_index: u64 = this + .read_immediate(&this.project_index(index, i)?)? + .to_scalar() + .to_u32()? + .into(); + let dest = this.project_index(&dest, i)?; + + let val = if src_index < left_len { + this.read_immediate(&this.project_index(&left, src_index)?)? + } else if src_index < left_len.checked_add(right_len).unwrap() { + let right_idx = src_index.checked_sub(left_len).unwrap(); + this.read_immediate(&this.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + this.write_immediate(*val, &dest)?; + } + } + "gather" => { + let [passthru, ptrs, mask] = check_arg_count(args)?; + let (passthru, passthru_len) = this.operand_to_simd(passthru)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, passthru_len); + assert_eq!(dest_len, ptrs_len); + assert_eq!(dest_len, mask_len); + + for i in 0..dest_len { + let passthru = this.read_immediate(&this.project_index(&passthru, i)?)?; + let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; + let mask = this.read_immediate(&this.project_index(&mask, i)?)?; + let dest = this.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + let place = this.deref_pointer(&ptr)?; + this.read_immediate(&place)? + } else { + passthru + }; + this.write_immediate(*val, &dest)?; + } + } + "scatter" => { + let [value, ptrs, mask] = check_arg_count(args)?; + let (value, value_len) = this.operand_to_simd(value)?; + let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + + assert_eq!(ptrs_len, value_len); + assert_eq!(ptrs_len, mask_len); + + for i in 0..ptrs_len { + let value = this.read_immediate(&this.project_index(&value, i)?)?; + let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; + let mask = this.read_immediate(&this.project_index(&mask, i)?)?; + + if simd_element_to_bool(mask)? { + let place = this.deref_pointer(&ptr)?; + this.write_immediate(*value, &place)?; + } + } + } + "masked_load" => { + let [mask, ptr, default] = check_arg_count(args)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let ptr = this.read_pointer(ptr)?; + let (default, default_len) = this.operand_to_simd(default)?; + let (dest, dest_len) = this.mplace_to_simd(dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, default_len); + + for i in 0..dest_len { + let mask = this.read_immediate(&this.project_index(&mask, i)?)?; + let default = this.read_immediate(&this.project_index(&default, i)?)?; + let dest = this.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + #[allow(clippy::arithmetic_side_effects)] + let ptr = ptr.wrapping_offset(dest.layout.size * i, this); + let place = this.ptr_to_mplace(ptr, dest.layout); + this.read_immediate(&place)? + } else { + default + }; + this.write_immediate(*val, &dest)?; + } + } + "masked_store" => { + let [mask, ptr, vals] = check_arg_count(args)?; + let (mask, mask_len) = this.operand_to_simd(mask)?; + let ptr = this.read_pointer(ptr)?; + let (vals, vals_len) = this.operand_to_simd(vals)?; + + assert_eq!(mask_len, vals_len); + + for i in 0..vals_len { + let mask = this.read_immediate(&this.project_index(&mask, i)?)?; + let val = this.read_immediate(&this.project_index(&vals, i)?)?; + + if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + #[allow(clippy::arithmetic_side_effects)] + let ptr = ptr.wrapping_offset(val.layout.size * i, this); + let place = this.ptr_to_mplace(ptr, val.layout); + this.write_immediate(*val, &place)? + }; + } + } + + name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"), + } + Ok(()) + } + + fn fminmax_op( + &self, + op: MinMax, + left: &ImmTy<'tcx, Provenance>, + right: &ImmTy<'tcx, Provenance>, + ) -> InterpResult<'tcx, Scalar> { + let this = self.eval_context_ref(); + assert_eq!(left.layout.ty, right.layout.ty); + let ty::Float(float_ty) = left.layout.ty.kind() else { + bug!("fmax operand is not a float") + }; + let left = left.to_scalar(); + let right = right.to_scalar(); + Ok(match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let left = left.to_f32()?; + let right = right.to_f32()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = this.adjust_nan(res, &[left, right]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let left = left.to_f64()?; + let right = right.to_f64()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = this.adjust_nan(res, &[left, right]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + }) + } +} + +fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 { + assert!(idx < vec_len); + match endianness { + Endian::Little => idx, + #[allow(clippy::arithmetic_side_effects)] // idx < vec_len + Endian::Big => vec_len - 1 - idx, // reverse order of bits + } +} diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs index f47c84842b1..f01c2da25d9 100644 --- a/src/tools/miri/src/lib.rs +++ b/src/tools/miri/src/lib.rs @@ -79,6 +79,7 @@ mod concurrency; mod diagnostics; mod eval; mod helpers; +mod intrinsics; mod machine; mod mono_hash_map; mod operator; @@ -95,9 +96,9 @@ pub use rustc_const_eval::interpret::*; #[doc(no_inline)] pub use rustc_const_eval::interpret::{self, AllocMap, PlaceTy, Provenance as _}; +pub use crate::intrinsics::EvalContextExt as _; pub use crate::shims::env::{EnvVars, EvalContextExt as _}; pub use crate::shims::foreign_items::{DynSym, EvalContextExt as _}; -pub use crate::shims::intrinsics::EvalContextExt as _; pub use crate::shims::os_str::EvalContextExt as _; pub use crate::shims::panic::{CatchUnwindData, EvalContextExt as _}; pub use crate::shims::time::EvalContextExt as _; diff --git a/src/tools/miri/src/shims/intrinsics/atomic.rs b/src/tools/miri/src/shims/intrinsics/atomic.rs deleted file mode 100644 index 865886a7fc1..00000000000 --- a/src/tools/miri/src/shims/intrinsics/atomic.rs +++ /dev/null @@ -1,288 +0,0 @@ -use rustc_middle::{mir, mir::BinOp, ty}; - -use crate::*; -use helpers::check_arg_count; - -pub enum AtomicOp { - /// The `bool` indicates whether the result of the operation should be negated - /// (must be a boolean-typed operation). - MirOp(mir::BinOp, bool), - Max, - Min, -} - -impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} -pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { - /// Calls the atomic intrinsic `intrinsic`; the `atomic_` prefix has already been removed. - fn emulate_atomic_intrinsic( - &mut self, - intrinsic_name: &str, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect(); - - fn read_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicReadOrd> { - Ok(match ord { - "seqcst" => AtomicReadOrd::SeqCst, - "acquire" => AtomicReadOrd::Acquire, - "relaxed" => AtomicReadOrd::Relaxed, - _ => throw_unsup_format!("unsupported read ordering `{ord}`"), - }) - } - - fn write_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicWriteOrd> { - Ok(match ord { - "seqcst" => AtomicWriteOrd::SeqCst, - "release" => AtomicWriteOrd::Release, - "relaxed" => AtomicWriteOrd::Relaxed, - _ => throw_unsup_format!("unsupported write ordering `{ord}`"), - }) - } - - fn rw_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicRwOrd> { - Ok(match ord { - "seqcst" => AtomicRwOrd::SeqCst, - "acqrel" => AtomicRwOrd::AcqRel, - "acquire" => AtomicRwOrd::Acquire, - "release" => AtomicRwOrd::Release, - "relaxed" => AtomicRwOrd::Relaxed, - _ => throw_unsup_format!("unsupported read-write ordering `{ord}`"), - }) - } - - fn fence_ord<'tcx>(ord: &str) -> InterpResult<'tcx, AtomicFenceOrd> { - Ok(match ord { - "seqcst" => AtomicFenceOrd::SeqCst, - "acqrel" => AtomicFenceOrd::AcqRel, - "acquire" => AtomicFenceOrd::Acquire, - "release" => AtomicFenceOrd::Release, - _ => throw_unsup_format!("unsupported fence ordering `{ord}`"), - }) - } - - match &*intrinsic_structure { - ["load", ord] => this.atomic_load(args, dest, read_ord(ord)?)?, - ["store", ord] => this.atomic_store(args, write_ord(ord)?)?, - - ["fence", ord] => this.atomic_fence_intrinsic(args, fence_ord(ord)?)?, - ["singlethreadfence", ord] => this.compiler_fence_intrinsic(args, fence_ord(ord)?)?, - - ["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord)?)?, - ["cxchg", ord1, ord2] => - this.atomic_compare_exchange(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?, - ["cxchgweak", ord1, ord2] => - this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1)?, read_ord(ord2)?)?, - - ["or", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord)?)?, - ["xor", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord)?)?, - ["and", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord)?)?, - ["nand", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord)?)?, - ["xadd", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord)?)?, - ["xsub", ord] => - this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord)?)?, - ["min", ord] => { - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); - this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?; - } - ["umin", ord] => { - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); - this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord)?)?; - } - ["max", ord] => { - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Int(_))); - this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?; - } - ["umax", ord] => { - // Later we will use the type to indicate signed vs unsigned, - // so make sure it matches the intrinsic name. - assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_))); - this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?; - } - - _ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"), - } - Ok(()) - } -} - -impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for MiriInterpCx<'mir, 'tcx> {} -trait EvalContextPrivExt<'mir, 'tcx: 'mir>: MiriInterpCxExt<'mir, 'tcx> { - fn atomic_load( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - atomic: AtomicReadOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - - // Perform atomic load. - let val = this.read_scalar_atomic(&place, atomic)?; - // Perform regular store. - this.write_scalar(val, dest)?; - Ok(()) - } - - fn atomic_store( - &mut self, - args: &[OpTy<'tcx, Provenance>], - atomic: AtomicWriteOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, val] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - - // Perform regular load. - let val = this.read_scalar(val)?; - // Perform atomic store - this.write_scalar_atomic(val, &place, atomic)?; - Ok(()) - } - - fn compiler_fence_intrinsic( - &mut self, - args: &[OpTy<'tcx, Provenance>], - atomic: AtomicFenceOrd, - ) -> InterpResult<'tcx> { - let [] = check_arg_count(args)?; - let _ = atomic; - //FIXME: compiler fences are currently ignored - Ok(()) - } - - fn atomic_fence_intrinsic( - &mut self, - args: &[OpTy<'tcx, Provenance>], - atomic: AtomicFenceOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - let [] = check_arg_count(args)?; - this.atomic_fence(atomic)?; - Ok(()) - } - - fn atomic_rmw_op( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - atomic_op: AtomicOp, - atomic: AtomicRwOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, rhs] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - let rhs = this.read_immediate(rhs)?; - - if !place.layout.ty.is_integral() && !place.layout.ty.is_unsafe_ptr() { - span_bug!( - this.cur_span(), - "atomic arithmetic operations only work on integer and raw pointer types", - ); - } - if rhs.layout.ty != place.layout.ty { - span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch"); - } - - match atomic_op { - AtomicOp::Min => { - let old = this.atomic_min_max_scalar(&place, rhs, true, atomic)?; - this.write_immediate(*old, dest)?; // old value is returned - Ok(()) - } - AtomicOp::Max => { - let old = this.atomic_min_max_scalar(&place, rhs, false, atomic)?; - this.write_immediate(*old, dest)?; // old value is returned - Ok(()) - } - AtomicOp::MirOp(op, neg) => { - let old = this.atomic_rmw_op_immediate(&place, &rhs, op, neg, atomic)?; - this.write_immediate(*old, dest)?; // old value is returned - Ok(()) - } - } - } - - fn atomic_exchange( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - atomic: AtomicRwOrd, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, new] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - let new = this.read_scalar(new)?; - - let old = this.atomic_exchange_scalar(&place, new, atomic)?; - this.write_scalar(old, dest)?; // old value is returned - Ok(()) - } - - fn atomic_compare_exchange_impl( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - can_fail_spuriously: bool, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - let [place, expect_old, new] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - let expect_old = this.read_immediate(expect_old)?; // read as immediate for the sake of `binary_op()` - let new = this.read_scalar(new)?; - - let old = this.atomic_compare_exchange_scalar( - &place, - &expect_old, - new, - success, - fail, - can_fail_spuriously, - )?; - - // Return old value. - this.write_immediate(old, dest)?; - Ok(()) - } - - fn atomic_compare_exchange( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - ) -> InterpResult<'tcx> { - self.atomic_compare_exchange_impl(args, dest, success, fail, false) - } - - fn atomic_compare_exchange_weak( - &mut self, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - success: AtomicRwOrd, - fail: AtomicReadOrd, - ) -> InterpResult<'tcx> { - self.atomic_compare_exchange_impl(args, dest, success, fail, true) - } -} diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs deleted file mode 100644 index a7ba4fd7f9e..00000000000 --- a/src/tools/miri/src/shims/intrinsics/mod.rs +++ /dev/null @@ -1,433 +0,0 @@ -mod atomic; -mod simd; - -use std::iter; - -use rand::Rng; -use rustc_apfloat::{Float, Round}; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::{ - mir, - ty::{self, FloatTy}, -}; -use rustc_target::abi::Size; - -use crate::*; -use atomic::EvalContextExt as _; -use helpers::{check_arg_count, ToHost, ToSoft}; -use simd::EvalContextExt as _; - -impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} -pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { - fn call_intrinsic( - &mut self, - instance: ty::Instance<'tcx>, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - ret: Option, - _unwind: mir::UnwindAction, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - // See if the core engine can handle this intrinsic. - if this.emulate_intrinsic(instance, args, dest, ret)? { - return Ok(()); - } - let intrinsic_name = this.tcx.item_name(instance.def_id()); - let intrinsic_name = intrinsic_name.as_str(); - - // Handle intrinsics without return place. - match intrinsic_name { - "abort" => { - throw_machine_stop!(TerminationInfo::Abort( - "the program aborted execution".to_owned() - )) - } - _ => {} - } - - // All remaining supported intrinsics have a return place. - let ret = match ret { - None => throw_unsup_format!("unimplemented (diverging) intrinsic: `{intrinsic_name}`"), - Some(p) => p, - }; - - // Some intrinsics are special and need the "ret". - match intrinsic_name { - "catch_unwind" => return this.handle_catch_unwind(args, dest, ret), - _ => {} - } - - // The rest jumps to `ret` immediately. - this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest)?; - - trace!("{:?}", this.dump_place(&dest.clone().into())); - this.go_to_block(ret); - Ok(()) - } - - /// Emulates a Miri-supported intrinsic (not supported by the core engine). - fn emulate_intrinsic_by_name( - &mut self, - intrinsic_name: &str, - generic_args: ty::GenericArgsRef<'tcx>, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - - if let Some(name) = intrinsic_name.strip_prefix("atomic_") { - return this.emulate_atomic_intrinsic(name, args, dest); - } - if let Some(name) = intrinsic_name.strip_prefix("simd_") { - return this.emulate_simd_intrinsic(name, generic_args, args, dest); - } - - match intrinsic_name { - // Miri overwriting CTFE intrinsics. - "ptr_guaranteed_cmp" => { - let [left, right] = check_arg_count(args)?; - let left = this.read_immediate(left)?; - let right = this.read_immediate(right)?; - let val = this.wrapping_binary_op(mir::BinOp::Eq, &left, &right)?; - // We're type punning a bool as an u8 here. - this.write_scalar(val.to_scalar(), dest)?; - } - "const_allocate" => { - // For now, for compatibility with the run-time implementation of this, we just return null. - // See . - this.write_null(dest)?; - } - "const_deallocate" => { - // complete NOP - } - - // Raw memory accesses - "volatile_load" => { - let [place] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - this.copy_op(&place, dest)?; - } - "volatile_store" => { - let [place, dest] = check_arg_count(args)?; - let place = this.deref_pointer(place)?; - this.copy_op(dest, &place)?; - } - - "write_bytes" | "volatile_set_memory" => { - let [ptr, val_byte, count] = check_arg_count(args)?; - let ty = ptr.layout.ty.builtin_deref(true).unwrap().ty; - let ty_layout = this.layout_of(ty)?; - let val_byte = this.read_scalar(val_byte)?.to_u8()?; - let ptr = this.read_pointer(ptr)?; - let count = this.read_target_usize(count)?; - // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max), - // but no actual allocation can be big enough for the difference to be noticeable. - let byte_count = ty_layout.size.checked_mul(count, this).ok_or_else(|| { - err_ub_format!("overflow computing total size of `{intrinsic_name}`") - })?; - this.write_bytes_ptr(ptr, iter::repeat(val_byte).take(byte_count.bytes_usize()))?; - } - - // Memory model / provenance manipulation - "ptr_mask" => { - let [ptr, mask] = check_arg_count(args)?; - - let ptr = this.read_pointer(ptr)?; - let mask = this.read_target_usize(mask)?; - - let masked_addr = Size::from_bytes(ptr.addr().bytes() & mask); - - this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?; - } - - // We want to return either `true` or `false` at random, or else something like - // ``` - // if !is_val_statically_known(0) { unreachable_unchecked(); } - // ``` - // Would not be considered UB, or the other way around (`is_val_statically_known(0)`). - "is_val_statically_known" => { - let [arg] = check_arg_count(args)?; - this.validate_operand(arg)?; - let branch: bool = this.machine.rng.get_mut().gen(); - this.write_scalar(Scalar::from_bool(branch), dest)?; - } - - // Floating-point operations - "fabsf32" => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f32()?; - // This is a "bitwise" operation, so there's no NaN non-determinism. - this.write_scalar(Scalar::from_f32(f.abs()), dest)?; - } - "fabsf64" => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f64()?; - // This is a "bitwise" operation, so there's no NaN non-determinism. - this.write_scalar(Scalar::from_f64(f.abs()), dest)?; - } - "floorf32" | "ceilf32" | "truncf32" | "roundf32" | "rintf32" => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f32()?; - let mode = match intrinsic_name { - "floorf32" => Round::TowardNegative, - "ceilf32" => Round::TowardPositive, - "truncf32" => Round::TowardZero, - "roundf32" => Round::NearestTiesToAway, - "rintf32" => Round::NearestTiesToEven, - _ => bug!(), - }; - let res = f.round_to_integral(mode).value; - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - #[rustfmt::skip] - | "sinf32" - | "cosf32" - | "sqrtf32" - | "expf32" - | "exp2f32" - | "logf32" - | "log10f32" - | "log2f32" - => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f32()?; - // Using host floats (but it's fine, these operations do not have guaranteed precision). - let f_host = f.to_host(); - let res = match intrinsic_name { - "sinf32" => f_host.sin(), - "cosf32" => f_host.cos(), - "sqrtf32" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats - "expf32" => f_host.exp(), - "exp2f32" => f_host.exp2(), - "logf32" => f_host.ln(), - "log10f32" => f_host.log10(), - "log2f32" => f_host.log2(), - _ => bug!(), - }; - let res = res.to_soft(); - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - - "floorf64" | "ceilf64" | "truncf64" | "roundf64" | "rintf64" => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f64()?; - let mode = match intrinsic_name { - "floorf64" => Round::TowardNegative, - "ceilf64" => Round::TowardPositive, - "truncf64" => Round::TowardZero, - "roundf64" => Round::NearestTiesToAway, - "rintf64" => Round::NearestTiesToEven, - _ => bug!(), - }; - let res = f.round_to_integral(mode).value; - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - #[rustfmt::skip] - | "sinf64" - | "cosf64" - | "sqrtf64" - | "expf64" - | "exp2f64" - | "logf64" - | "log10f64" - | "log2f64" - => { - let [f] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f64()?; - // Using host floats (but it's fine, these operations do not have guaranteed precision). - let f_host = f.to_host(); - let res = match intrinsic_name { - "sinf64" => f_host.sin(), - "cosf64" => f_host.cos(), - "sqrtf64" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats - "expf64" => f_host.exp(), - "exp2f64" => f_host.exp2(), - "logf64" => f_host.ln(), - "log10f64" => f_host.log10(), - "log2f64" => f_host.log2(), - _ => bug!(), - }; - let res = res.to_soft(); - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - - #[rustfmt::skip] - | "fadd_fast" - | "fsub_fast" - | "fmul_fast" - | "fdiv_fast" - | "frem_fast" - => { - let [a, b] = check_arg_count(args)?; - let a = this.read_immediate(a)?; - let b = this.read_immediate(b)?; - let op = match intrinsic_name { - "fadd_fast" => mir::BinOp::Add, - "fsub_fast" => mir::BinOp::Sub, - "fmul_fast" => mir::BinOp::Mul, - "fdiv_fast" => mir::BinOp::Div, - "frem_fast" => mir::BinOp::Rem, - _ => bug!(), - }; - let float_finite = |x: &ImmTy<'tcx, _>| -> InterpResult<'tcx, bool> { - let ty::Float(fty) = x.layout.ty.kind() else { - bug!("float_finite: non-float input type {}", x.layout.ty) - }; - Ok(match fty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => x.to_scalar().to_f32()?.is_finite(), - FloatTy::F64 => x.to_scalar().to_f64()?.is_finite(), - FloatTy::F128 => unimplemented!("f16_f128"), - }) - }; - match (float_finite(&a)?, float_finite(&b)?) { - (false, false) => throw_ub_format!( - "`{intrinsic_name}` intrinsic called with non-finite value as both parameters", - ), - (false, _) => throw_ub_format!( - "`{intrinsic_name}` intrinsic called with non-finite value as first parameter", - ), - (_, false) => throw_ub_format!( - "`{intrinsic_name}` intrinsic called with non-finite value as second parameter", - ), - _ => {} - } - let res = this.wrapping_binary_op(op, &a, &b)?; - if !float_finite(&res)? { - throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result"); - } - // This cannot be a NaN so we also don't have to apply any non-determinism. - // (Also, `wrapping_binary_op` already called `generate_nan` if needed.) - this.write_immediate(*res, dest)?; - } - - #[rustfmt::skip] - | "minnumf32" - | "maxnumf32" - | "copysignf32" - => { - let [a, b] = check_arg_count(args)?; - let a = this.read_scalar(a)?.to_f32()?; - let b = this.read_scalar(b)?.to_f32()?; - let res = match intrinsic_name { - "minnumf32" => this.adjust_nan(a.min(b), &[a, b]), - "maxnumf32" => this.adjust_nan(a.max(b), &[a, b]), - "copysignf32" => a.copy_sign(b), // bitwise, no NaN adjustments - _ => bug!(), - }; - this.write_scalar(Scalar::from_f32(res), dest)?; - } - - #[rustfmt::skip] - | "minnumf64" - | "maxnumf64" - | "copysignf64" - => { - let [a, b] = check_arg_count(args)?; - let a = this.read_scalar(a)?.to_f64()?; - let b = this.read_scalar(b)?.to_f64()?; - let res = match intrinsic_name { - "minnumf64" => this.adjust_nan(a.min(b), &[a, b]), - "maxnumf64" => this.adjust_nan(a.max(b), &[a, b]), - "copysignf64" => a.copy_sign(b), // bitwise, no NaN adjustments - _ => bug!(), - }; - this.write_scalar(Scalar::from_f64(res), dest)?; - } - - "fmaf32" => { - let [a, b, c] = check_arg_count(args)?; - let a = this.read_scalar(a)?.to_f32()?; - let b = this.read_scalar(b)?.to_f32()?; - let c = this.read_scalar(c)?.to_f32()?; - // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11 - let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); - let res = this.adjust_nan(res, &[a, b, c]); - this.write_scalar(res, dest)?; - } - - "fmaf64" => { - let [a, b, c] = check_arg_count(args)?; - let a = this.read_scalar(a)?.to_f64()?; - let b = this.read_scalar(b)?.to_f64()?; - let c = this.read_scalar(c)?.to_f64()?; - // FIXME: Using host floats, to work around https://github.com/rust-lang/rustc_apfloat/issues/11 - let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); - let res = this.adjust_nan(res, &[a, b, c]); - this.write_scalar(res, dest)?; - } - - "powf32" => { - let [f1, f2] = check_arg_count(args)?; - let f1 = this.read_scalar(f1)?.to_f32()?; - let f2 = this.read_scalar(f2)?.to_f32()?; - // Using host floats (but it's fine, this operation does not have guaranteed precision). - let res = f1.to_host().powf(f2.to_host()).to_soft(); - let res = this.adjust_nan(res, &[f1, f2]); - this.write_scalar(res, dest)?; - } - - "powf64" => { - let [f1, f2] = check_arg_count(args)?; - let f1 = this.read_scalar(f1)?.to_f64()?; - let f2 = this.read_scalar(f2)?.to_f64()?; - // Using host floats (but it's fine, this operation does not have guaranteed precision). - let res = f1.to_host().powf(f2.to_host()).to_soft(); - let res = this.adjust_nan(res, &[f1, f2]); - this.write_scalar(res, dest)?; - } - - "powif32" => { - let [f, i] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f32()?; - let i = this.read_scalar(i)?.to_i32()?; - // Using host floats (but it's fine, this operation does not have guaranteed precision). - let res = f.to_host().powi(i).to_soft(); - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - - "powif64" => { - let [f, i] = check_arg_count(args)?; - let f = this.read_scalar(f)?.to_f64()?; - let i = this.read_scalar(i)?.to_i32()?; - // Using host floats (but it's fine, this operation does not have guaranteed precision). - let res = f.to_host().powi(i).to_soft(); - let res = this.adjust_nan(res, &[f]); - this.write_scalar(res, dest)?; - } - - "float_to_int_unchecked" => { - let [val] = check_arg_count(args)?; - let val = this.read_immediate(val)?; - - let res = this - .float_to_int_checked(&val, dest.layout, Round::TowardZero)? - .ok_or_else(|| { - err_ub_format!( - "`float_to_int_unchecked` intrinsic called on {val} which cannot be represented in target type `{:?}`", - dest.layout.ty - ) - })?; - - this.write_immediate(*res, dest)?; - } - - // Other - "breakpoint" => { - let [] = check_arg_count(args)?; - // normally this would raise a SIGTRAP, which aborts if no debugger is connected - throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap"))) - } - - name => throw_unsup_format!("unimplemented intrinsic: `{name}`"), - } - - Ok(()) - } -} diff --git a/src/tools/miri/src/shims/intrinsics/simd.rs b/src/tools/miri/src/shims/intrinsics/simd.rs deleted file mode 100644 index e6d6f72404e..00000000000 --- a/src/tools/miri/src/shims/intrinsics/simd.rs +++ /dev/null @@ -1,798 +0,0 @@ -use rustc_apfloat::{Float, Round}; -use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; -use rustc_middle::{mir, ty, ty::FloatTy}; -use rustc_span::{sym, Symbol}; -use rustc_target::abi::{Endian, HasDataLayout}; - -use crate::helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool, ToHost, ToSoft}; -use crate::*; - -#[derive(Copy, Clone)] -pub(crate) enum MinMax { - Min, - Max, -} - -impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriInterpCx<'mir, 'tcx> {} -pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { - /// Calls the simd intrinsic `intrinsic`; the `simd_` prefix has already been removed. - fn emulate_simd_intrinsic( - &mut self, - intrinsic_name: &str, - generic_args: ty::GenericArgsRef<'tcx>, - args: &[OpTy<'tcx, Provenance>], - dest: &MPlaceTy<'tcx, Provenance>, - ) -> InterpResult<'tcx> { - let this = self.eval_context_mut(); - match intrinsic_name { - #[rustfmt::skip] - | "neg" - | "fabs" - | "ceil" - | "floor" - | "round" - | "trunc" - | "fsqrt" - | "fsin" - | "fcos" - | "fexp" - | "fexp2" - | "flog" - | "flog2" - | "flog10" - | "ctlz" - | "cttz" - | "bswap" - | "bitreverse" - => { - let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, op_len); - - #[derive(Copy, Clone)] - enum Op<'a> { - MirOp(mir::UnOp), - Abs, - Round(rustc_apfloat::Round), - Numeric(Symbol), - HostOp(&'a str), - } - let which = match intrinsic_name { - "neg" => Op::MirOp(mir::UnOp::Neg), - "fabs" => Op::Abs, - "ceil" => Op::Round(rustc_apfloat::Round::TowardPositive), - "floor" => Op::Round(rustc_apfloat::Round::TowardNegative), - "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway), - "trunc" => Op::Round(rustc_apfloat::Round::TowardZero), - "ctlz" => Op::Numeric(sym::ctlz), - "cttz" => Op::Numeric(sym::cttz), - "bswap" => Op::Numeric(sym::bswap), - "bitreverse" => Op::Numeric(sym::bitreverse), - _ => Op::HostOp(intrinsic_name), - }; - - for i in 0..dest_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - let dest = this.project_index(&dest, i)?; - let val = match which { - Op::MirOp(mir_op) => { - // This already does NaN adjustments - this.wrapping_unary_op(mir_op, &op)?.to_scalar() - } - Op::Abs => { - // Works for f32 and f64. - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) - }; - let op = op.to_scalar(); - // "Bitwise" operation, no NaN adjustments - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()), - FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()), - FloatTy::F128 => unimplemented!("f16_f128"), - } - } - Op::HostOp(host_op) => { - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) - }; - // Using host floats (but it's fine, these operations do not have guaranteed precision). - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let f = op.to_scalar().to_f32()?; - let f_host = f.to_host(); - let res = match host_op { - "fsqrt" => f_host.sqrt(), // FIXME Using host floats, this should use full-precision soft-floats - "fsin" => f_host.sin(), - "fcos" => f_host.cos(), - "fexp" => f_host.exp(), - "fexp2" => f_host.exp2(), - "flog" => f_host.ln(), - "flog2" => f_host.log2(), - "flog10" => f_host.log10(), - _ => bug!(), - }; - let res = res.to_soft(); - let res = this.adjust_nan(res, &[f]); - Scalar::from(res) - } - FloatTy::F64 => { - let f = op.to_scalar().to_f64()?; - let f_host = f.to_host(); - let res = match host_op { - "fsqrt" => f_host.sqrt(), - "fsin" => f_host.sin(), - "fcos" => f_host.cos(), - "fexp" => f_host.exp(), - "fexp2" => f_host.exp2(), - "flog" => f_host.ln(), - "flog2" => f_host.log2(), - "flog10" => f_host.log10(), - _ => bug!(), - }; - let res = res.to_soft(); - let res = this.adjust_nan(res, &[f]); - Scalar::from(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - } - } - Op::Round(rounding) => { - let ty::Float(float_ty) = op.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) - }; - match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let f = op.to_scalar().to_f32()?; - let res = f.round_to_integral(rounding).value; - let res = this.adjust_nan(res, &[f]); - Scalar::from_f32(res) - } - FloatTy::F64 => { - let f = op.to_scalar().to_f64()?; - let res = f.round_to_integral(rounding).value; - let res = this.adjust_nan(res, &[f]); - Scalar::from_f64(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - } - } - Op::Numeric(name) => { - this.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? - } - }; - this.write_scalar(val, &dest)?; - } - } - #[rustfmt::skip] - | "add" - | "sub" - | "mul" - | "div" - | "rem" - | "shl" - | "shr" - | "and" - | "or" - | "xor" - | "eq" - | "ne" - | "lt" - | "le" - | "gt" - | "ge" - | "fmax" - | "fmin" - | "saturating_add" - | "saturating_sub" - | "arith_offset" - => { - use mir::BinOp; - - let [left, right] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, left_len); - assert_eq!(dest_len, right_len); - - enum Op { - MirOp(BinOp), - SaturatingOp(BinOp), - FMinMax(MinMax), - WrappingOffset, - } - let which = match intrinsic_name { - "add" => Op::MirOp(BinOp::Add), - "sub" => Op::MirOp(BinOp::Sub), - "mul" => Op::MirOp(BinOp::Mul), - "div" => Op::MirOp(BinOp::Div), - "rem" => Op::MirOp(BinOp::Rem), - "shl" => Op::MirOp(BinOp::Shl), - "shr" => Op::MirOp(BinOp::Shr), - "and" => Op::MirOp(BinOp::BitAnd), - "or" => Op::MirOp(BinOp::BitOr), - "xor" => Op::MirOp(BinOp::BitXor), - "eq" => Op::MirOp(BinOp::Eq), - "ne" => Op::MirOp(BinOp::Ne), - "lt" => Op::MirOp(BinOp::Lt), - "le" => Op::MirOp(BinOp::Le), - "gt" => Op::MirOp(BinOp::Gt), - "ge" => Op::MirOp(BinOp::Ge), - "fmax" => Op::FMinMax(MinMax::Max), - "fmin" => Op::FMinMax(MinMax::Min), - "saturating_add" => Op::SaturatingOp(BinOp::Add), - "saturating_sub" => Op::SaturatingOp(BinOp::Sub), - "arith_offset" => Op::WrappingOffset, - _ => unreachable!(), - }; - - for i in 0..dest_len { - let left = this.read_immediate(&this.project_index(&left, i)?)?; - let right = this.read_immediate(&this.project_index(&right, i)?)?; - let dest = this.project_index(&dest, i)?; - let val = match which { - Op::MirOp(mir_op) => { - // This does NaN adjustments. - let (val, overflowed) = this.overflowing_binary_op(mir_op, &left, &right)?; - if matches!(mir_op, BinOp::Shl | BinOp::Shr) { - // Shifts have extra UB as SIMD operations that the MIR binop does not have. - // See . - if overflowed { - let r_val = right.to_scalar().to_bits(right.layout.size)?; - throw_ub_format!("overflowing shift by {r_val} in `simd_{intrinsic_name}` in SIMD lane {i}"); - } - } - if matches!(mir_op, BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge) { - // Special handling for boolean-returning operations - assert_eq!(val.layout.ty, this.tcx.types.bool); - let val = val.to_scalar().to_bool().unwrap(); - bool_to_simd_element(val, dest.layout.size) - } else { - assert_ne!(val.layout.ty, this.tcx.types.bool); - assert_eq!(val.layout.ty, dest.layout.ty); - val.to_scalar() - } - } - Op::SaturatingOp(mir_op) => { - this.saturating_arith(mir_op, &left, &right)? - } - Op::WrappingOffset => { - let ptr = left.to_scalar().to_pointer(this)?; - let offset_count = right.to_scalar().to_target_isize(this)?; - let pointee_ty = left.layout.ty.builtin_deref(true).unwrap().ty; - - let pointee_size = i64::try_from(this.layout_of(pointee_ty)?.size.bytes()).unwrap(); - let offset_bytes = offset_count.wrapping_mul(pointee_size); - let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, this); - Scalar::from_maybe_pointer(offset_ptr, this) - } - Op::FMinMax(op) => { - this.fminmax_op(op, &left, &right)? - } - }; - this.write_scalar(val, &dest)?; - } - } - "fma" => { - let [a, b, c] = check_arg_count(args)?; - let (a, a_len) = this.operand_to_simd(a)?; - let (b, b_len) = this.operand_to_simd(b)?; - let (c, c_len) = this.operand_to_simd(c)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, a_len); - assert_eq!(dest_len, b_len); - assert_eq!(dest_len, c_len); - - for i in 0..dest_len { - let a = this.read_scalar(&this.project_index(&a, i)?)?; - let b = this.read_scalar(&this.project_index(&b, i)?)?; - let c = this.read_scalar(&this.project_index(&c, i)?)?; - let dest = this.project_index(&dest, i)?; - - // Works for f32 and f64. - // FIXME: using host floats to work around https://github.com/rust-lang/miri/issues/2468. - let ty::Float(float_ty) = dest.layout.ty.kind() else { - span_bug!(this.cur_span(), "{} operand is not a float", intrinsic_name) - }; - let val = match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let a = a.to_f32()?; - let b = b.to_f32()?; - let c = c.to_f32()?; - let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); - let res = this.adjust_nan(res, &[a, b, c]); - Scalar::from(res) - } - FloatTy::F64 => { - let a = a.to_f64()?; - let b = b.to_f64()?; - let c = c.to_f64()?; - let res = a.to_host().mul_add(b.to_host(), c.to_host()).to_soft(); - let res = this.adjust_nan(res, &[a, b, c]); - Scalar::from(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - }; - this.write_scalar(val, &dest)?; - } - } - #[rustfmt::skip] - | "reduce_and" - | "reduce_or" - | "reduce_xor" - | "reduce_any" - | "reduce_all" - | "reduce_max" - | "reduce_min" => { - use mir::BinOp; - - let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - - let imm_from_bool = - |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool); - - enum Op { - MirOp(BinOp), - MirOpBool(BinOp), - MinMax(MinMax), - } - let which = match intrinsic_name { - "reduce_and" => Op::MirOp(BinOp::BitAnd), - "reduce_or" => Op::MirOp(BinOp::BitOr), - "reduce_xor" => Op::MirOp(BinOp::BitXor), - "reduce_any" => Op::MirOpBool(BinOp::BitOr), - "reduce_all" => Op::MirOpBool(BinOp::BitAnd), - "reduce_max" => Op::MinMax(MinMax::Max), - "reduce_min" => Op::MinMax(MinMax::Min), - _ => unreachable!(), - }; - - // Initialize with first lane, then proceed with the rest. - let mut res = this.read_immediate(&this.project_index(&op, 0)?)?; - if matches!(which, Op::MirOpBool(_)) { - // Convert to `bool` scalar. - res = imm_from_bool(simd_element_to_bool(res)?); - } - for i in 1..op_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - res = match which { - Op::MirOp(mir_op) => { - this.wrapping_binary_op(mir_op, &res, &op)? - } - Op::MirOpBool(mir_op) => { - let op = imm_from_bool(simd_element_to_bool(op)?); - this.wrapping_binary_op(mir_op, &res, &op)? - } - Op::MinMax(mmop) => { - if matches!(res.layout.ty.kind(), ty::Float(_)) { - ImmTy::from_scalar(this.fminmax_op(mmop, &res, &op)?, res.layout) - } else { - // Just boring integers, so NaNs to worry about - let mirop = match mmop { - MinMax::Min => BinOp::Le, - MinMax::Max => BinOp::Ge, - }; - if this.wrapping_binary_op(mirop, &res, &op)?.to_scalar().to_bool()? { - res - } else { - op - } - } - } - }; - } - this.write_immediate(*res, dest)?; - } - #[rustfmt::skip] - | "reduce_add_ordered" - | "reduce_mul_ordered" => { - use mir::BinOp; - - let [op, init] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let init = this.read_immediate(init)?; - - let mir_op = match intrinsic_name { - "reduce_add_ordered" => BinOp::Add, - "reduce_mul_ordered" => BinOp::Mul, - _ => unreachable!(), - }; - - let mut res = init; - for i in 0..op_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - res = this.wrapping_binary_op(mir_op, &res, &op)?; - } - this.write_immediate(*res, dest)?; - } - "select" => { - let [mask, yes, no] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (yes, yes_len) = this.operand_to_simd(yes)?; - let (no, no_len) = this.operand_to_simd(no)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, mask_len); - assert_eq!(dest_len, yes_len); - assert_eq!(dest_len, no_len); - - for i in 0..dest_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let yes = this.read_immediate(&this.project_index(&yes, i)?)?; - let no = this.read_immediate(&this.project_index(&no, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { yes } else { no }; - this.write_immediate(*val, &dest)?; - } - } - // Variant of `select` that takes a bitmask rather than a "vector of bool". - "select_bitmask" => { - let [mask, yes, no] = check_arg_count(args)?; - let (yes, yes_len) = this.operand_to_simd(yes)?; - let (no, no_len) = this.operand_to_simd(no)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - let bitmask_len = dest_len.next_multiple_of(8); - - // The mask must be an integer or an array. - assert!( - mask.layout.ty.is_integral() - || matches!(mask.layout.ty.kind(), ty::Array(elemty, _) if elemty == &this.tcx.types.u8) - ); - assert!(bitmask_len <= 64); - assert_eq!(bitmask_len, mask.layout.size.bits()); - assert_eq!(dest_len, yes_len); - assert_eq!(dest_len, no_len); - let dest_len = u32::try_from(dest_len).unwrap(); - let bitmask_len = u32::try_from(bitmask_len).unwrap(); - - // To read the mask, we transmute it to an integer. - // That does the right thing wrt endianness. - let mask_ty = this.machine.layouts.uint(mask.layout.size).unwrap(); - let mask = mask.transmute(mask_ty, this)?; - let mask: u64 = this.read_scalar(&mask)?.to_bits(mask_ty.size)?.try_into().unwrap(); - - for i in 0..dest_len { - let bit_i = simd_bitmask_index(i, dest_len, this.data_layout().endian); - let mask = mask & 1u64.checked_shl(bit_i).unwrap(); - let yes = this.read_immediate(&this.project_index(&yes, i.into())?)?; - let no = this.read_immediate(&this.project_index(&no, i.into())?)?; - let dest = this.project_index(&dest, i.into())?; - - let val = if mask != 0 { yes } else { no }; - this.write_immediate(*val, &dest)?; - } - for i in dest_len..bitmask_len { - // If the mask is "padded", ensure that padding is all-zero. - // This deliberately does not use `simd_bitmask_index`; these bits are outside - // the bitmask. It does not matter in which order we check them. - let mask = mask & 1u64.checked_shl(i).unwrap(); - if mask != 0 { - throw_ub_format!( - "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits" - ); - } - } - } - // Converts a "vector of bool" into a bitmask. - "bitmask" => { - let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let bitmask_len = op_len.next_multiple_of(8); - - // Returns either an unsigned integer or array of `u8`. - assert!( - dest.layout.ty.is_integral() - || matches!(dest.layout.ty.kind(), ty::Array(elemty, _) if elemty == &this.tcx.types.u8) - ); - assert!(bitmask_len <= 64); - assert_eq!(bitmask_len, dest.layout.size.bits()); - let op_len = u32::try_from(op_len).unwrap(); - - let mut res = 0u64; - for i in 0..op_len { - let op = this.read_immediate(&this.project_index(&op, i.into())?)?; - if simd_element_to_bool(op)? { - res |= 1u64 - .checked_shl(simd_bitmask_index(i, op_len, this.data_layout().endian)) - .unwrap(); - } - } - // We have to change the type of the place to be able to write `res` into it. This - // transmutes the integer to an array, which does the right thing wrt endianness. - let dest = - dest.transmute(this.machine.layouts.uint(dest.layout.size).unwrap(), this)?; - this.write_int(res, &dest)?; - } - "cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => { - let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, op_len); - - let unsafe_cast = intrinsic_name == "cast"; - let safe_cast = intrinsic_name == "as"; - let ptr_cast = intrinsic_name == "cast_ptr"; - let expose_cast = intrinsic_name == "expose_provenance"; - let from_exposed_cast = intrinsic_name == "with_exposed_provenance"; - - for i in 0..dest_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { - // Int-to-(int|float): always safe - (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) - if safe_cast || unsafe_cast => - this.int_to_int_or_float(&op, dest.layout)?, - // Float-to-float: always safe - (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast => - this.float_to_float_or_int(&op, dest.layout)?, - // Float-to-int in safe mode - (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast => - this.float_to_float_or_int(&op, dest.layout)?, - // Float-to-int in unchecked mode - (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => { - this.float_to_int_checked(&op, dest.layout, Round::TowardZero)? - .ok_or_else(|| { - err_ub_format!( - "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`", - dest.layout.ty - ) - })? - } - // Ptr-to-ptr cast - (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast => - this.ptr_to_ptr(&op, dest.layout)?, - // Ptr/Int casts - (ty::RawPtr(..), ty::Int(_) | ty::Uint(_)) if expose_cast => - this.pointer_expose_provenance_cast(&op, dest.layout)?, - (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast => - this.pointer_with_exposed_provenance_cast(&op, dest.layout)?, - // Error otherwise - _ => - throw_unsup_format!( - "Unsupported SIMD cast from element type {from_ty} to {to_ty}", - from_ty = op.layout.ty, - to_ty = dest.layout.ty, - ), - }; - this.write_immediate(*val, &dest)?; - } - } - "shuffle_generic" => { - let [left, right] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - let index = generic_args[2] - .expect_const() - .eval(*this.tcx, this.param_env(), this.tcx.span) - .unwrap() - .unwrap_branch(); - let index_len = index.len(); - - assert_eq!(left_len, right_len); - assert_eq!(index_len as u64, dest_len); - - for i in 0..dest_len { - let src_index: u64 = index[usize::try_from(i).unwrap()] - .unwrap_leaf() - .try_to_u32() - .unwrap() - .into(); - let dest = this.project_index(&dest, i)?; - - let val = if src_index < left_len { - this.read_immediate(&this.project_index(&left, src_index)?)? - } else if src_index < left_len.checked_add(right_len).unwrap() { - let right_idx = src_index.checked_sub(left_len).unwrap(); - this.read_immediate(&this.project_index(&right, right_idx)?)? - } else { - throw_ub_format!( - "`simd_shuffle_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" - ); - }; - this.write_immediate(*val, &dest)?; - } - } - "shuffle" => { - let [left, right, index] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - // `index` is an array, not a SIMD type - let ty::Array(_, index_len) = index.layout.ty.kind() else { - span_bug!( - this.cur_span(), - "simd_shuffle index argument has non-array type {}", - index.layout.ty - ) - }; - let index_len = index_len.eval_target_usize(*this.tcx, this.param_env()); - - assert_eq!(left_len, right_len); - assert_eq!(index_len, dest_len); - - for i in 0..dest_len { - let src_index: u64 = this - .read_immediate(&this.project_index(index, i)?)? - .to_scalar() - .to_u32()? - .into(); - let dest = this.project_index(&dest, i)?; - - let val = if src_index < left_len { - this.read_immediate(&this.project_index(&left, src_index)?)? - } else if src_index < left_len.checked_add(right_len).unwrap() { - let right_idx = src_index.checked_sub(left_len).unwrap(); - this.read_immediate(&this.project_index(&right, right_idx)?)? - } else { - throw_ub_format!( - "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" - ); - }; - this.write_immediate(*val, &dest)?; - } - } - "gather" => { - let [passthru, ptrs, mask] = check_arg_count(args)?; - let (passthru, passthru_len) = this.operand_to_simd(passthru)?; - let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, passthru_len); - assert_eq!(dest_len, ptrs_len); - assert_eq!(dest_len, mask_len); - - for i in 0..dest_len { - let passthru = this.read_immediate(&this.project_index(&passthru, i)?)?; - let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { - let place = this.deref_pointer(&ptr)?; - this.read_immediate(&place)? - } else { - passthru - }; - this.write_immediate(*val, &dest)?; - } - } - "scatter" => { - let [value, ptrs, mask] = check_arg_count(args)?; - let (value, value_len) = this.operand_to_simd(value)?; - let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - - assert_eq!(ptrs_len, value_len); - assert_eq!(ptrs_len, mask_len); - - for i in 0..ptrs_len { - let value = this.read_immediate(&this.project_index(&value, i)?)?; - let ptr = this.read_immediate(&this.project_index(&ptrs, i)?)?; - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - - if simd_element_to_bool(mask)? { - let place = this.deref_pointer(&ptr)?; - this.write_immediate(*value, &place)?; - } - } - } - "masked_load" => { - let [mask, ptr, default] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let ptr = this.read_pointer(ptr)?; - let (default, default_len) = this.operand_to_simd(default)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; - - assert_eq!(dest_len, mask_len); - assert_eq!(dest_len, default_len); - - for i in 0..dest_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let default = this.read_immediate(&this.project_index(&default, i)?)?; - let dest = this.project_index(&dest, i)?; - - let val = if simd_element_to_bool(mask)? { - // Size * u64 is implemented as always checked - #[allow(clippy::arithmetic_side_effects)] - let ptr = ptr.wrapping_offset(dest.layout.size * i, this); - let place = this.ptr_to_mplace(ptr, dest.layout); - this.read_immediate(&place)? - } else { - default - }; - this.write_immediate(*val, &dest)?; - } - } - "masked_store" => { - let [mask, ptr, vals] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let ptr = this.read_pointer(ptr)?; - let (vals, vals_len) = this.operand_to_simd(vals)?; - - assert_eq!(mask_len, vals_len); - - for i in 0..vals_len { - let mask = this.read_immediate(&this.project_index(&mask, i)?)?; - let val = this.read_immediate(&this.project_index(&vals, i)?)?; - - if simd_element_to_bool(mask)? { - // Size * u64 is implemented as always checked - #[allow(clippy::arithmetic_side_effects)] - let ptr = ptr.wrapping_offset(val.layout.size * i, this); - let place = this.ptr_to_mplace(ptr, val.layout); - this.write_immediate(*val, &place)? - }; - } - } - - name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"), - } - Ok(()) - } - - fn fminmax_op( - &self, - op: MinMax, - left: &ImmTy<'tcx, Provenance>, - right: &ImmTy<'tcx, Provenance>, - ) -> InterpResult<'tcx, Scalar> { - let this = self.eval_context_ref(); - assert_eq!(left.layout.ty, right.layout.ty); - let ty::Float(float_ty) = left.layout.ty.kind() else { - bug!("fmax operand is not a float") - }; - let left = left.to_scalar(); - let right = right.to_scalar(); - Ok(match float_ty { - FloatTy::F16 => unimplemented!("f16_f128"), - FloatTy::F32 => { - let left = left.to_f32()?; - let right = right.to_f32()?; - let res = match op { - MinMax::Min => left.min(right), - MinMax::Max => left.max(right), - }; - let res = this.adjust_nan(res, &[left, right]); - Scalar::from_f32(res) - } - FloatTy::F64 => { - let left = left.to_f64()?; - let right = right.to_f64()?; - let res = match op { - MinMax::Min => left.min(right), - MinMax::Max => left.max(right), - }; - let res = this.adjust_nan(res, &[left, right]); - Scalar::from_f64(res) - } - FloatTy::F128 => unimplemented!("f16_f128"), - }) - } -} - -fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 { - assert!(idx < vec_len); - match endianness { - Endian::Little => idx, - #[allow(clippy::arithmetic_side_effects)] // idx < vec_len - Endian::Big => vec_len - 1 - idx, // reverse order of bits - } -} diff --git a/src/tools/miri/src/shims/mod.rs b/src/tools/miri/src/shims/mod.rs index 85c9a202f7d..ab5a8c5877e 100644 --- a/src/tools/miri/src/shims/mod.rs +++ b/src/tools/miri/src/shims/mod.rs @@ -5,7 +5,6 @@ mod backtrace; #[cfg(target_os = "linux")] pub mod ffi_support; pub mod foreign_items; -pub mod intrinsics; pub mod unix; pub mod windows; mod x86; diff --git a/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs new file mode 100644 index 00000000000..9263ad381d1 --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.rs @@ -0,0 +1,42 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// We cannot use `cfg`-based tricks here since the output would be +// different for non-x86 targets. +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +// Explicitly disable SSE4.1 because it is enabled by default on macOS +//@compile-flags: -C target-feature=-sse4.1 + +#![feature(link_llvm_intrinsics, simd_ffi)] + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +fn main() { + assert!(is_x86_feature_detected!("sse")); + assert!(!is_x86_feature_detected!("sse4.1")); + + unsafe { + // Pass, since SSE is enabled + addss(_mm_setzero_ps(), _mm_setzero_ps()); + + // Fail, since SSE4.1 is not enabled + dpps(_mm_setzero_ps(), _mm_setzero_ps(), 0); + //~^ ERROR: Undefined Behavior: attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 + } +} + +#[allow(improper_ctypes)] +extern "C" { + #[link_name = "llvm.x86.sse.add.ss"] + fn addss(a: __m128, b: __m128) -> __m128; + + #[link_name = "llvm.x86.sse41.dpps"] + fn dpps(a: __m128, b: __m128, imm8: u8) -> __m128; +} diff --git a/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr new file mode 100644 index 00000000000..8e83d20854f --- /dev/null +++ b/src/tools/miri/tests/fail/intrinsics/intrinsic_target_feature.stderr @@ -0,0 +1,15 @@ +error: Undefined Behavior: attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 + --> $DIR/intrinsic_target_feature.rs:LL:CC + | +LL | dpps(_mm_setzero_ps(), _mm_setzero_ps(), 0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: BACKTRACE: + = note: inside `main` at $DIR/intrinsic_target_feature.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs b/src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs deleted file mode 100644 index 9263ad381d1..00000000000 --- a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// We cannot use `cfg`-based tricks here since the output would be -// different for non-x86 targets. -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -// Explicitly disable SSE4.1 because it is enabled by default on macOS -//@compile-flags: -C target-feature=-sse4.1 - -#![feature(link_llvm_intrinsics, simd_ffi)] - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - -fn main() { - assert!(is_x86_feature_detected!("sse")); - assert!(!is_x86_feature_detected!("sse4.1")); - - unsafe { - // Pass, since SSE is enabled - addss(_mm_setzero_ps(), _mm_setzero_ps()); - - // Fail, since SSE4.1 is not enabled - dpps(_mm_setzero_ps(), _mm_setzero_ps(), 0); - //~^ ERROR: Undefined Behavior: attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 - } -} - -#[allow(improper_ctypes)] -extern "C" { - #[link_name = "llvm.x86.sse.add.ss"] - fn addss(a: __m128, b: __m128) -> __m128; - - #[link_name = "llvm.x86.sse41.dpps"] - fn dpps(a: __m128, b: __m128, imm8: u8) -> __m128; -} diff --git a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr b/src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr deleted file mode 100644 index 8e83d20854f..00000000000 --- a/src/tools/miri/tests/fail/shims/intrinsic_target_feature.stderr +++ /dev/null @@ -1,15 +0,0 @@ -error: Undefined Behavior: attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 - --> $DIR/intrinsic_target_feature.rs:LL:CC - | -LL | dpps(_mm_setzero_ps(), _mm_setzero_ps(), 0); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempted to call intrinsic `llvm.x86.sse41.dpps` that requires missing target feature sse4.1 - | - = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior - = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information - = note: BACKTRACE: - = note: inside `main` at $DIR/intrinsic_target_feature.rs:LL:CC - -note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace - -error: aborting due to 1 previous error - diff --git a/src/tools/miri/tests/pass/intrinsics-integer.rs b/src/tools/miri/tests/pass/intrinsics-integer.rs deleted file mode 100644 index 13e7bd8e1b9..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-integer.rs +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org) - -#![feature(core_intrinsics)] -use std::intrinsics::*; - -pub fn main() { - unsafe { - [assert_eq!(ctpop(0u8), 0), assert_eq!(ctpop(0i8), 0)]; - [assert_eq!(ctpop(0u16), 0), assert_eq!(ctpop(0i16), 0)]; - [assert_eq!(ctpop(0u32), 0), assert_eq!(ctpop(0i32), 0)]; - [assert_eq!(ctpop(0u64), 0), assert_eq!(ctpop(0i64), 0)]; - - [assert_eq!(ctpop(1u8), 1), assert_eq!(ctpop(1i8), 1)]; - [assert_eq!(ctpop(1u16), 1), assert_eq!(ctpop(1i16), 1)]; - [assert_eq!(ctpop(1u32), 1), assert_eq!(ctpop(1i32), 1)]; - [assert_eq!(ctpop(1u64), 1), assert_eq!(ctpop(1i64), 1)]; - - [assert_eq!(ctpop(10u8), 2), assert_eq!(ctpop(10i8), 2)]; - [assert_eq!(ctpop(10u16), 2), assert_eq!(ctpop(10i16), 2)]; - [assert_eq!(ctpop(10u32), 2), assert_eq!(ctpop(10i32), 2)]; - [assert_eq!(ctpop(10u64), 2), assert_eq!(ctpop(10i64), 2)]; - - [assert_eq!(ctpop(100u8), 3), assert_eq!(ctpop(100i8), 3)]; - [assert_eq!(ctpop(100u16), 3), assert_eq!(ctpop(100i16), 3)]; - [assert_eq!(ctpop(100u32), 3), assert_eq!(ctpop(100i32), 3)]; - [assert_eq!(ctpop(100u64), 3), assert_eq!(ctpop(100i64), 3)]; - - [assert_eq!(ctpop(-1i8 as u8), 8), assert_eq!(ctpop(-1i8), 8)]; - [assert_eq!(ctpop(-1i16 as u16), 16), assert_eq!(ctpop(-1i16), 16)]; - [assert_eq!(ctpop(-1i32 as u32), 32), assert_eq!(ctpop(-1i32), 32)]; - [assert_eq!(ctpop(-1i64 as u64), 64), assert_eq!(ctpop(-1i64), 64)]; - - [assert_eq!(ctlz(0u8), 8), assert_eq!(ctlz(0i8), 8)]; - [assert_eq!(ctlz(0u16), 16), assert_eq!(ctlz(0i16), 16)]; - [assert_eq!(ctlz(0u32), 32), assert_eq!(ctlz(0i32), 32)]; - [assert_eq!(ctlz(0u64), 64), assert_eq!(ctlz(0i64), 64)]; - - [assert_eq!(ctlz(1u8), 7), assert_eq!(ctlz(1i8), 7)]; - [assert_eq!(ctlz(1u16), 15), assert_eq!(ctlz(1i16), 15)]; - [assert_eq!(ctlz(1u32), 31), assert_eq!(ctlz(1i32), 31)]; - [assert_eq!(ctlz(1u64), 63), assert_eq!(ctlz(1i64), 63)]; - - [assert_eq!(ctlz(10u8), 4), assert_eq!(ctlz(10i8), 4)]; - [assert_eq!(ctlz(10u16), 12), assert_eq!(ctlz(10i16), 12)]; - [assert_eq!(ctlz(10u32), 28), assert_eq!(ctlz(10i32), 28)]; - [assert_eq!(ctlz(10u64), 60), assert_eq!(ctlz(10i64), 60)]; - - [assert_eq!(ctlz(100u8), 1), assert_eq!(ctlz(100i8), 1)]; - [assert_eq!(ctlz(100u16), 9), assert_eq!(ctlz(100i16), 9)]; - [assert_eq!(ctlz(100u32), 25), assert_eq!(ctlz(100i32), 25)]; - [assert_eq!(ctlz(100u64), 57), assert_eq!(ctlz(100i64), 57)]; - - [assert_eq!(ctlz_nonzero(1u8), 7), assert_eq!(ctlz_nonzero(1i8), 7)]; - [assert_eq!(ctlz_nonzero(1u16), 15), assert_eq!(ctlz_nonzero(1i16), 15)]; - [assert_eq!(ctlz_nonzero(1u32), 31), assert_eq!(ctlz_nonzero(1i32), 31)]; - [assert_eq!(ctlz_nonzero(1u64), 63), assert_eq!(ctlz_nonzero(1i64), 63)]; - - [assert_eq!(ctlz_nonzero(10u8), 4), assert_eq!(ctlz_nonzero(10i8), 4)]; - [assert_eq!(ctlz_nonzero(10u16), 12), assert_eq!(ctlz_nonzero(10i16), 12)]; - [assert_eq!(ctlz_nonzero(10u32), 28), assert_eq!(ctlz_nonzero(10i32), 28)]; - [assert_eq!(ctlz_nonzero(10u64), 60), assert_eq!(ctlz_nonzero(10i64), 60)]; - - [assert_eq!(ctlz_nonzero(100u8), 1), assert_eq!(ctlz_nonzero(100i8), 1)]; - [assert_eq!(ctlz_nonzero(100u16), 9), assert_eq!(ctlz_nonzero(100i16), 9)]; - [assert_eq!(ctlz_nonzero(100u32), 25), assert_eq!(ctlz_nonzero(100i32), 25)]; - [assert_eq!(ctlz_nonzero(100u64), 57), assert_eq!(ctlz_nonzero(100i64), 57)]; - - [assert_eq!(cttz(-1i8 as u8), 0), assert_eq!(cttz(-1i8), 0)]; - [assert_eq!(cttz(-1i16 as u16), 0), assert_eq!(cttz(-1i16), 0)]; - [assert_eq!(cttz(-1i32 as u32), 0), assert_eq!(cttz(-1i32), 0)]; - [assert_eq!(cttz(-1i64 as u64), 0), assert_eq!(cttz(-1i64), 0)]; - - [assert_eq!(cttz(0u8), 8), assert_eq!(cttz(0i8), 8)]; - [assert_eq!(cttz(0u16), 16), assert_eq!(cttz(0i16), 16)]; - [assert_eq!(cttz(0u32), 32), assert_eq!(cttz(0i32), 32)]; - [assert_eq!(cttz(0u64), 64), assert_eq!(cttz(0i64), 64)]; - - [assert_eq!(cttz(1u8), 0), assert_eq!(cttz(1i8), 0)]; - [assert_eq!(cttz(1u16), 0), assert_eq!(cttz(1i16), 0)]; - [assert_eq!(cttz(1u32), 0), assert_eq!(cttz(1i32), 0)]; - [assert_eq!(cttz(1u64), 0), assert_eq!(cttz(1i64), 0)]; - - [assert_eq!(cttz(10u8), 1), assert_eq!(cttz(10i8), 1)]; - [assert_eq!(cttz(10u16), 1), assert_eq!(cttz(10i16), 1)]; - [assert_eq!(cttz(10u32), 1), assert_eq!(cttz(10i32), 1)]; - [assert_eq!(cttz(10u64), 1), assert_eq!(cttz(10i64), 1)]; - - [assert_eq!(cttz(100u8), 2), assert_eq!(cttz(100i8), 2)]; - [assert_eq!(cttz(100u16), 2), assert_eq!(cttz(100i16), 2)]; - [assert_eq!(cttz(100u32), 2), assert_eq!(cttz(100i32), 2)]; - [assert_eq!(cttz(100u64), 2), assert_eq!(cttz(100i64), 2)]; - - [assert_eq!(cttz_nonzero(-1i8 as u8), 0), assert_eq!(cttz_nonzero(-1i8), 0)]; - [assert_eq!(cttz_nonzero(-1i16 as u16), 0), assert_eq!(cttz_nonzero(-1i16), 0)]; - [assert_eq!(cttz_nonzero(-1i32 as u32), 0), assert_eq!(cttz_nonzero(-1i32), 0)]; - [assert_eq!(cttz_nonzero(-1i64 as u64), 0), assert_eq!(cttz_nonzero(-1i64), 0)]; - - [assert_eq!(cttz_nonzero(1u8), 0), assert_eq!(cttz_nonzero(1i8), 0)]; - [assert_eq!(cttz_nonzero(1u16), 0), assert_eq!(cttz_nonzero(1i16), 0)]; - [assert_eq!(cttz_nonzero(1u32), 0), assert_eq!(cttz_nonzero(1i32), 0)]; - [assert_eq!(cttz_nonzero(1u64), 0), assert_eq!(cttz_nonzero(1i64), 0)]; - - [assert_eq!(cttz_nonzero(10u8), 1), assert_eq!(cttz_nonzero(10i8), 1)]; - [assert_eq!(cttz_nonzero(10u16), 1), assert_eq!(cttz_nonzero(10i16), 1)]; - [assert_eq!(cttz_nonzero(10u32), 1), assert_eq!(cttz_nonzero(10i32), 1)]; - [assert_eq!(cttz_nonzero(10u64), 1), assert_eq!(cttz_nonzero(10i64), 1)]; - - [assert_eq!(cttz_nonzero(100u8), 2), assert_eq!(cttz_nonzero(100i8), 2)]; - [assert_eq!(cttz_nonzero(100u16), 2), assert_eq!(cttz_nonzero(100i16), 2)]; - [assert_eq!(cttz_nonzero(100u32), 2), assert_eq!(cttz_nonzero(100i32), 2)]; - [assert_eq!(cttz_nonzero(100u64), 2), assert_eq!(cttz_nonzero(100i64), 2)]; - - assert_eq!(bswap(0x0Au8), 0x0A); // no-op - assert_eq!(bswap(0x0Ai8), 0x0A); // no-op - assert_eq!(bswap(0x0A0Bu16), 0x0B0A); - assert_eq!(bswap(0x0A0Bi16), 0x0B0A); - assert_eq!(bswap(0x0ABBCC0Du32), 0x0DCCBB0A); - assert_eq!(bswap(0x0ABBCC0Di32), 0x0DCCBB0A); - assert_eq!(bswap(0x0122334455667708u64), 0x0877665544332201); - assert_eq!(bswap(0x0122334455667708i64), 0x0877665544332201); - - assert_eq!(exact_div(9 * 9u32, 3), 27); - assert_eq!(exact_div(-9 * 9i32, 3), -27); - assert_eq!(exact_div(9 * 9i8, -3), -27); - assert_eq!(exact_div(-9 * 9i64, -3), 27); - - assert_eq!(unchecked_div(9 * 9u32, 2), 40); - assert_eq!(unchecked_div(-9 * 9i32, 2), -40); - assert_eq!(unchecked_div(9 * 9i8, -2), -40); - assert_eq!(unchecked_div(-9 * 9i64, -2), 40); - - assert_eq!(unchecked_rem(9 * 9u32, 2), 1); - assert_eq!(unchecked_rem(-9 * 9i32, 2), -1); - assert_eq!(unchecked_rem(9 * 9i8, -2), 1); - assert_eq!(unchecked_rem(-9 * 9i64, -2), -1); - - assert_eq!(unchecked_add(23u8, 19), 42); - assert_eq!(unchecked_add(5, -10), -5); - - assert_eq!(unchecked_sub(23u8, 19), 4); - assert_eq!(unchecked_sub(-17, -27), 10); - - assert_eq!(unchecked_mul(6u8, 7), 42); - assert_eq!(unchecked_mul(13, -5), -65); - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs b/src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs deleted file mode 100644 index 7363c753617..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-aes-vaes.rs +++ /dev/null @@ -1,291 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=+aes,+vaes,+avx512f - -#![feature(avx512_target_feature, stdarch_x86_avx512)] - -use core::mem::transmute; -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - -fn main() { - assert!(is_x86_feature_detected!("aes")); - assert!(is_x86_feature_detected!("vaes")); - assert!(is_x86_feature_detected!("avx512f")); - - unsafe { - test_aes(); - test_vaes(); - } -} - -// The constants in the tests below are just bit patterns. They should not -// be interpreted as integers; signedness does not make sense for them, but -// __m128i happens to be defined in terms of signed integers. -#[allow(overflowing_literals)] -#[target_feature(enable = "aes")] -unsafe fn test_aes() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/aes.rs - - #[target_feature(enable = "aes")] - unsafe fn test_mm_aesdec_si128() { - // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx. - let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); - let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); - let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee); - let r = _mm_aesdec_si128(a, k); - assert_eq_m128i(r, e); - } - test_mm_aesdec_si128(); - - #[target_feature(enable = "aes")] - unsafe fn test_mm_aesdeclast_si128() { - // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx. - let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); - let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); - let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493); - let r = _mm_aesdeclast_si128(a, k); - assert_eq_m128i(r, e); - } - test_mm_aesdeclast_si128(); - - #[target_feature(enable = "aes")] - unsafe fn test_mm_aesenc_si128() { - // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx. - let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); - let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); - let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333); - let r = _mm_aesenc_si128(a, k); - assert_eq_m128i(r, e); - } - test_mm_aesenc_si128(); - - #[target_feature(enable = "aes")] - unsafe fn test_mm_aesenclast_si128() { - // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx. - let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); - let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); - let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8); - let r = _mm_aesenclast_si128(a, k); - assert_eq_m128i(r, e); - } - test_mm_aesenclast_si128(); - - #[target_feature(enable = "aes")] - unsafe fn test_mm_aesimc_si128() { - // Constants taken from https://msdn.microsoft.com/en-us/library/cc714195.aspx. - let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); - let e = _mm_set_epi64x(0xc66c82284ee40aa0, 0x6633441122770055); - let r = _mm_aesimc_si128(a); - assert_eq_m128i(r, e); - } - test_mm_aesimc_si128(); -} - -// The constants in the tests below are just bit patterns. They should not -// be interpreted as integers; signedness does not make sense for them, but -// __m128i happens to be defined in terms of signed integers. -#[allow(overflowing_literals)] -#[target_feature(enable = "vaes,avx512f")] -unsafe fn test_vaes() { - #[target_feature(enable = "avx")] - unsafe fn get_a256() -> __m256i { - // Constants are random - _mm256_set_epi64x( - 0xb89f43a558d3cd51, - 0x57b3e81e369bd603, - 0xf177a1a626933fd6, - 0x50d8adbed1a2f9d7, - ) - } - #[target_feature(enable = "avx")] - unsafe fn get_k256() -> __m256i { - // Constants are random - _mm256_set_epi64x( - 0x503ff704588b5627, - 0xe23d882ed9c3c146, - 0x2785e5b670155b3c, - 0xa750718e183549ff, - ) - } - - #[target_feature(enable = "vaes")] - unsafe fn test_mm256_aesdec_epi128() { - let a = get_a256(); - let k = get_k256(); - let r = _mm256_aesdec_epi128(a, k); - - // Check results. - let a: [u128; 2] = transmute(a); - let k: [u128; 2] = transmute(k); - let r: [u128; 2] = transmute(r); - for i in 0..2 { - let e: u128 = transmute(_mm_aesdec_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm256_aesdec_epi128(); - - #[target_feature(enable = "vaes")] - unsafe fn test_mm256_aesdeclast_epi128() { - let a = get_a256(); - let k = get_k256(); - let r = _mm256_aesdeclast_epi128(a, k); - - // Check results. - let a: [u128; 2] = transmute(a); - let k: [u128; 2] = transmute(k); - let r: [u128; 2] = transmute(r); - for i in 0..2 { - let e: u128 = transmute(_mm_aesdeclast_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm256_aesdeclast_epi128(); - - #[target_feature(enable = "vaes")] - unsafe fn test_mm256_aesenc_epi128() { - let a = get_a256(); - let k = get_k256(); - let r = _mm256_aesenc_epi128(a, k); - - // Check results. - let a: [u128; 2] = transmute(a); - let k: [u128; 2] = transmute(k); - let r: [u128; 2] = transmute(r); - for i in 0..2 { - let e: u128 = transmute(_mm_aesenc_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm256_aesenc_epi128(); - - #[target_feature(enable = "vaes")] - unsafe fn test_mm256_aesenclast_epi128() { - let a = get_a256(); - let k = get_k256(); - let r = _mm256_aesenclast_epi128(a, k); - - // Check results. - let a: [u128; 2] = transmute(a); - let k: [u128; 2] = transmute(k); - let r: [u128; 2] = transmute(r); - for i in 0..2 { - let e: u128 = transmute(_mm_aesenclast_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm256_aesenclast_epi128(); - - #[target_feature(enable = "avx512f")] - unsafe fn get_a512() -> __m512i { - // Constants are random - _mm512_set_epi64( - 0xb89f43a558d3cd51, - 0x57b3e81e369bd603, - 0xf177a1a626933fd6, - 0x50d8adbed1a2f9d7, - 0xfbfee3116629db78, - 0x6aef4a91f2ad50f4, - 0x4258bb51ff1d476d, - 0x31da65761c8016cf, - ) - } - #[target_feature(enable = "avx512f")] - unsafe fn get_k512() -> __m512i { - // Constants are random - _mm512_set_epi64( - 0x503ff704588b5627, - 0xe23d882ed9c3c146, - 0x2785e5b670155b3c, - 0xa750718e183549ff, - 0xdfb408830a65d3d9, - 0x0de3d92adac81b0a, - 0xed2741fe12877cae, - 0x3251ddb5404e0974, - ) - } - - #[target_feature(enable = "vaes,avx512f")] - unsafe fn test_mm512_aesdec_epi128() { - let a = get_a512(); - let k = get_k512(); - let r = _mm512_aesdec_epi128(a, k); - - // Check results. - let a: [u128; 4] = transmute(a); - let k: [u128; 4] = transmute(k); - let r: [u128; 4] = transmute(r); - for i in 0..4 { - let e: u128 = transmute(_mm_aesdec_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm512_aesdec_epi128(); - - #[target_feature(enable = "vaes,avx512f")] - unsafe fn test_mm512_aesdeclast_epi128() { - let a = get_a512(); - let k = get_k512(); - let r = _mm512_aesdeclast_epi128(a, k); - - // Check results. - let a: [u128; 4] = transmute(a); - let k: [u128; 4] = transmute(k); - let r: [u128; 4] = transmute(r); - for i in 0..4 { - let e: u128 = transmute(_mm_aesdeclast_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm512_aesdeclast_epi128(); - - #[target_feature(enable = "vaes,avx512f")] - unsafe fn test_mm512_aesenc_epi128() { - let a = get_a512(); - let k = get_k512(); - let r = _mm512_aesenc_epi128(a, k); - - // Check results. - let a: [u128; 4] = transmute(a); - let k: [u128; 4] = transmute(k); - let r: [u128; 4] = transmute(r); - for i in 0..4 { - let e: u128 = transmute(_mm_aesenc_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm512_aesenc_epi128(); - - #[target_feature(enable = "vaes,avx512f")] - unsafe fn test_mm512_aesenclast_epi128() { - let a = get_a512(); - let k = get_k512(); - let r = _mm512_aesenclast_epi128(a, k); - - // Check results. - let a: [u128; 4] = transmute(a); - let k: [u128; 4] = transmute(k); - let r: [u128; 4] = transmute(r); - for i in 0..4 { - let e: u128 = transmute(_mm_aesenclast_si128(transmute(a[i]), transmute(k[i]))); - assert_eq!(r[i], e); - } - } - test_mm512_aesenclast_epi128(); -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx.rs b/src/tools/miri/tests/pass/intrinsics-x86-avx.rs deleted file mode 100644 index 7d43cc596ae..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-avx.rs +++ /dev/null @@ -1,1431 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=+avx - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; -use std::mem::transmute; - -fn main() { - assert!(is_x86_feature_detected!("avx")); - - unsafe { - test_avx(); - } -} - -#[target_feature(enable = "avx")] -unsafe fn test_avx() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx.rs - - macro_rules! assert_approx_eq { - ($a:expr, $b:expr, $eps:expr) => {{ - let (a, b) = (&$a, &$b); - assert!( - (*a - *b).abs() < $eps, - "assertion failed: `(left !== right)` \ - (left: `{:?}`, right: `{:?}`, expect diff: `{:?}`, real diff: `{:?}`)", - *a, - *b, - $eps, - (*a - *b).abs() - ); - }}; - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_max_pd() { - let a = _mm256_setr_pd(1., 4., 5., 8.); - let b = _mm256_setr_pd(2., 3., 6., 7.); - let r = _mm256_max_pd(a, b); - let e = _mm256_setr_pd(2., 4., 6., 8.); - assert_eq_m256d(r, e); - // > If the values being compared are both 0.0s (of either sign), the - // > value in the second operand (source operand) is returned. - let w = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); - let x = _mm256_max_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); - let wu: [u64; 4] = transmute(w); - let xu: [u64; 4] = transmute(x); - assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); - assert_eq!(xu, [0u64; 4]); - // > If only one value is a NaN (SNaN or QNaN) for this instruction, the - // > second operand (source operand), either a NaN or a valid - // > floating-point value, is written to the result. - let y = _mm256_max_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); - let z = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); - let yf: [f64; 4] = transmute(y); - let zf: [f64; 4] = transmute(z); - assert_eq!(yf, [0.0; 4]); - assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); - } - test_mm256_max_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_max_ps() { - let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); - let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_max_ps(a, b); - let e = _mm256_setr_ps(2., 4., 6., 8., 10., 12., 14., 16.); - assert_eq_m256(r, e); - // > If the values being compared are both 0.0s (of either sign), the - // > value in the second operand (source operand) is returned. - let w = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); - let x = _mm256_max_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); - let wu: [u32; 8] = transmute(w); - let xu: [u32; 8] = transmute(x); - assert_eq!(wu, [0x8000_0000u32; 8]); - assert_eq!(xu, [0u32; 8]); - // > If only one value is a NaN (SNaN or QNaN) for this instruction, the - // > second operand (source operand), either a NaN or a valid - // > floating-point value, is written to the result. - let y = _mm256_max_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); - let z = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); - let yf: [f32; 8] = transmute(y); - let zf: [f32; 8] = transmute(z); - assert_eq!(yf, [0.0; 8]); - assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); - } - test_mm256_max_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_min_pd() { - let a = _mm256_setr_pd(1., 4., 5., 8.); - let b = _mm256_setr_pd(2., 3., 6., 7.); - let r = _mm256_min_pd(a, b); - let e = _mm256_setr_pd(1., 3., 5., 7.); - assert_eq_m256d(r, e); - // > If the values being compared are both 0.0s (of either sign), the - // > value in the second operand (source operand) is returned. - let w = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); - let x = _mm256_min_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); - let wu: [u64; 4] = transmute(w); - let xu: [u64; 4] = transmute(x); - assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); - assert_eq!(xu, [0u64; 4]); - // > If only one value is a NaN (SNaN or QNaN) for this instruction, the - // > second operand (source operand), either a NaN or a valid - // > floating-point value, is written to the result. - let y = _mm256_min_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); - let z = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); - let yf: [f64; 4] = transmute(y); - let zf: [f64; 4] = transmute(z); - assert_eq!(yf, [0.0; 4]); - assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); - } - test_mm256_min_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_min_ps() { - let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); - let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); - let r = _mm256_min_ps(a, b); - let e = _mm256_setr_ps(1., 3., 5., 7., 9., 11., 13., 15.); - assert_eq_m256(r, e); - // > If the values being compared are both 0.0s (of either sign), the - // > value in the second operand (source operand) is returned. - let w = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); - let x = _mm256_min_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); - let wu: [u32; 8] = transmute(w); - let xu: [u32; 8] = transmute(x); - assert_eq!(wu, [0x8000_0000u32; 8]); - assert_eq!(xu, [0u32; 8]); - // > If only one value is a NaN (SNaN or QNaN) for this instruction, the - // > second operand (source operand), either a NaN or a valid - // > floating-point value, is written to the result. - let y = _mm256_min_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); - let z = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); - let yf: [f32; 8] = transmute(y); - let zf: [f32; 8] = transmute(z); - assert_eq!(yf, [0.0; 8]); - assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); - } - test_mm256_min_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_nearest_f32() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f32, res: f32) { - let a = _mm256_set1_ps(x); - let e = _mm256_set1_ps(res); - let r = _mm256_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m256(r, e); - // Assume round-to-nearest by default - let r = _mm256_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m256(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); - let e = _mm256_setr_ps(2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0); - let r = _mm256_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m256(r, e); - // Assume round-to-nearest by default - let r = _mm256_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m256(r, e); - } - test_round_nearest_f32(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_floor_f32() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f32, res: f32) { - let a = _mm256_set1_ps(x); - let e = _mm256_set1_ps(res); - let r = _mm256_floor_ps(a); - assert_eq_m256(r, e); - let r = _mm256_round_ps::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m256(r, e); - } - - // Test rounding direction - test(-2.5, -3.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -2.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); - let e = _mm256_setr_ps(1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0); - let r = _mm256_floor_ps(a); - assert_eq_m256(r, e); - let r = _mm256_round_ps::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m256(r, e); - } - test_round_floor_f32(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_ceil_f32() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f32, res: f32) { - let a = _mm256_set1_ps(x); - let e = _mm256_set1_ps(res); - let r = _mm256_ceil_ps(a); - assert_eq_m256(r, e); - let r = _mm256_round_ps::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m256(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 2.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 3.0); - - // Test that each element is rounded - let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); - let e = _mm256_setr_ps(2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0); - let r = _mm256_ceil_ps(a); - assert_eq_m256(r, e); - let r = _mm256_round_ps::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m256(r, e); - } - test_round_ceil_f32(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_trunc_f32() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f32, res: f32) { - let a = _mm256_set1_ps(x); - let e = _mm256_set1_ps(res); - let r = _mm256_round_ps::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m256(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); - let e = _mm256_setr_ps(1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0); - let r = _mm256_round_ps::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m256(r, e); - } - test_round_trunc_f32(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_nearest_f64() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f64, res: f64) { - let a = _mm256_set1_pd(x); - let e = _mm256_set1_pd(res); - let r = _mm256_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m256d(r, e); - // Assume round-to-nearest by default - let r = _mm256_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m256d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); - let e = _mm256_setr_pd(2.0, 4.0, 6.0, 8.0); - let r = _mm256_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m256d(r, e); - // Assume round-to-nearest by default - let r = _mm256_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m256d(r, e); - } - test_round_nearest_f64(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_floor_f64() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f64, res: f64) { - let a = _mm256_set1_pd(x); - let e = _mm256_set1_pd(res); - let r = _mm256_floor_pd(a); - assert_eq_m256d(r, e); - let r = _mm256_round_pd::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m256d(r, e); - } - - // Test rounding direction - test(-2.5, -3.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -2.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); - let e = _mm256_setr_pd(1.0, 3.0, 5.0, 7.0); - let r = _mm256_floor_pd(a); - assert_eq_m256d(r, e); - let r = _mm256_round_pd::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m256d(r, e); - } - test_round_floor_f64(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_ceil_f64() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f64, res: f64) { - let a = _mm256_set1_pd(x); - let e = _mm256_set1_pd(res); - let r = _mm256_ceil_pd(a); - assert_eq_m256d(r, e); - let r = _mm256_round_pd::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m256d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 2.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 3.0); - - // Test that each element is rounded - let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); - let e = _mm256_setr_pd(2.0, 4.0, 6.0, 8.0); - let r = _mm256_ceil_pd(a); - assert_eq_m256d(r, e); - let r = _mm256_round_pd::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m256d(r, e); - } - test_round_ceil_f64(); - - #[target_feature(enable = "avx")] - unsafe fn test_round_trunc_f64() { - #[target_feature(enable = "avx")] - unsafe fn test(x: f64, res: f64) { - let a = _mm256_set1_pd(x); - let e = _mm256_set1_pd(res); - let r = _mm256_round_pd::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m256d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); - let e = _mm256_setr_pd(1.0, 3.0, 5.0, 7.0); - let r = _mm256_round_pd::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m256d(r, e); - } - test_round_trunc_f64(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_sqrt_ps() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let r = _mm256_sqrt_ps(a); - let e = _mm256_setr_ps(2., 3., 4., 5., 2., 3., 4., 5.); - assert_eq_m256(r, e); - } - test_mm256_sqrt_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_rcp_ps() { - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm256_rcp_ps(a); - #[rustfmt::skip] - let e = _mm256_setr_ps( - 0.99975586, 0.49987793, 0.33325195, 0.24993896, - 0.19995117, 0.16662598, 0.14282227, 0.12496948, - ); - let rel_err = 0.00048828125; - - let r: [f32; 8] = transmute(r); - let e: [f32; 8] = transmute(e); - for i in 0..8 { - assert_approx_eq!(r[i], e[i], 2. * rel_err); - } - } - test_mm256_rcp_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_rsqrt_ps() { - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let r = _mm256_rsqrt_ps(a); - #[rustfmt::skip] - let e = _mm256_setr_ps( - 0.99975586, 0.7069092, 0.5772705, 0.49987793, - 0.44714355, 0.40820313, 0.3779297, 0.3534546, - ); - let rel_err = 0.00048828125; - - let r: [f32; 8] = transmute(r); - let e: [f32; 8] = transmute(e); - for i in 0..8 { - assert_approx_eq!(r[i], e[i], 2. * rel_err); - } - } - test_mm256_rsqrt_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_dp_ps() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_dp_ps::<0xFF>(a, b); - let e = _mm256_setr_ps(200., 200., 200., 200., 2387., 2387., 2387., 2387.); - assert_eq_m256(r, e); - } - test_mm256_dp_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_hadd_pd() { - let a = _mm256_setr_pd(4., 9., 16., 25.); - let b = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_hadd_pd(a, b); - let e = _mm256_setr_pd(13., 7., 41., 7.); - assert_eq_m256d(r, e); - - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_hadd_pd(a, b); - let e = _mm256_setr_pd(3., 11., 7., 15.); - assert_eq_m256d(r, e); - } - test_mm256_hadd_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_hadd_ps() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_hadd_ps(a, b); - let e = _mm256_setr_ps(13., 41., 7., 7., 13., 41., 17., 114.); - assert_eq_m256(r, e); - - let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); - let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_hadd_ps(a, b); - let e = _mm256_setr_ps(3., 7., 11., 15., 3., 7., 11., 15.); - assert_eq_m256(r, e); - } - test_mm256_hadd_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_hsub_pd() { - let a = _mm256_setr_pd(4., 9., 16., 25.); - let b = _mm256_setr_pd(4., 3., 2., 5.); - let r = _mm256_hsub_pd(a, b); - let e = _mm256_setr_pd(-5., 1., -9., -3.); - assert_eq_m256d(r, e); - - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_hsub_pd(a, b); - let e = _mm256_setr_pd(-1., -1., -1., -1.); - assert_eq_m256d(r, e); - } - test_mm256_hsub_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_hsub_ps() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let r = _mm256_hsub_ps(a, b); - let e = _mm256_setr_ps(-5., -9., 1., -3., -5., -9., -1., 14.); - assert_eq_m256(r, e); - - let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); - let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_hsub_ps(a, b); - let e = _mm256_setr_ps(-1., -1., -1., -1., -1., -1., -1., -1.); - assert_eq_m256(r, e); - } - test_mm256_hsub_ps(); - - fn expected_cmp(imm: i32, lhs: F, rhs: F, if_t: F, if_f: F) -> F { - let res = match imm { - _CMP_EQ_OQ => lhs == rhs, - _CMP_LT_OS => lhs < rhs, - _CMP_LE_OS => lhs <= rhs, - _CMP_UNORD_Q => lhs.partial_cmp(&rhs).is_none(), - _CMP_NEQ_UQ => lhs != rhs, - _CMP_NLT_UQ => !(lhs < rhs), - _CMP_NLE_UQ => !(lhs <= rhs), - _CMP_ORD_Q => lhs.partial_cmp(&rhs).is_some(), - _CMP_EQ_UQ => lhs == rhs || lhs.partial_cmp(&rhs).is_none(), - _CMP_NGE_US => !(lhs >= rhs), - _CMP_NGT_US => !(lhs > rhs), - _CMP_FALSE_OQ => false, - _CMP_NEQ_OQ => lhs != rhs && lhs.partial_cmp(&rhs).is_some(), - _CMP_GE_OS => lhs >= rhs, - _CMP_GT_OS => lhs > rhs, - _CMP_TRUE_US => true, - _ => unreachable!(), - }; - if res { if_t } else { if_f } - } - fn expected_cmp_f32(imm: i32, lhs: f32, rhs: f32) -> f32 { - expected_cmp(imm, lhs, rhs, f32::from_bits(u32::MAX), 0.0) - } - fn expected_cmp_f64(imm: i32, lhs: f64, rhs: f64) -> f64 { - expected_cmp(imm, lhs, rhs, f64::from_bits(u64::MAX), 0.0) - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm_cmp_ss() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f32::NAN, 0.0), - (0.0, f32::NAN), - (f32::NAN, f32::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm_setr_ps(lhs, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(rhs, 5.0, 6.0, 7.0); - let r: [u32; 4] = transmute(_mm_cmp_ss::(a, b)); - let e: [u32; 4] = - transmute(_mm_setr_ps(expected_cmp_f32(IMM, lhs, rhs), 2.0, 3.0, 4.0)); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm_cmp_ps() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f32::NAN, 0.0), - (0.0, f32::NAN), - (f32::NAN, f32::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm_set1_ps(lhs); - let b = _mm_set1_ps(rhs); - let r: [u32; 4] = transmute(_mm_cmp_ps::(a, b)); - let e: [u32; 4] = transmute(_mm_set1_ps(expected_cmp_f32(IMM, lhs, rhs))); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm_cmp_sd() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f64::NAN, 0.0), - (0.0, f64::NAN), - (f64::NAN, f64::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm_setr_pd(lhs, 2.0); - let b = _mm_setr_pd(rhs, 3.0); - let r: [u64; 2] = transmute(_mm_cmp_sd::(a, b)); - let e: [u64; 2] = transmute(_mm_setr_pd(expected_cmp_f64(IMM, lhs, rhs), 2.0)); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm_cmp_pd() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f64::NAN, 0.0), - (0.0, f64::NAN), - (f64::NAN, f64::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm_set1_pd(lhs); - let b = _mm_set1_pd(rhs); - let r: [u64; 2] = transmute(_mm_cmp_pd::(a, b)); - let e: [u64; 2] = transmute(_mm_set1_pd(expected_cmp_f64(IMM, lhs, rhs))); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cmp_ps() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f32::NAN, 0.0), - (0.0, f32::NAN), - (f32::NAN, f32::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm256_set1_ps(lhs); - let b = _mm256_set1_ps(rhs); - let r: [u32; 8] = transmute(_mm256_cmp_ps::(a, b)); - let e: [u32; 8] = transmute(_mm256_set1_ps(expected_cmp_f32(IMM, lhs, rhs))); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cmp_pd() { - let values = [ - (1.0, 1.0), - (0.0, 1.0), - (1.0, 0.0), - (f64::NAN, 0.0), - (0.0, f64::NAN), - (f64::NAN, f64::NAN), - ]; - - for (lhs, rhs) in values { - let a = _mm256_set1_pd(lhs); - let b = _mm256_set1_pd(rhs); - let r: [u64; 4] = transmute(_mm256_cmp_pd::(a, b)); - let e: [u64; 4] = transmute(_mm256_set1_pd(expected_cmp_f64(IMM, lhs, rhs))); - assert_eq!(r, e); - } - } - - #[target_feature(enable = "avx")] - unsafe fn test_cmp() { - test_mm_cmp_ss::(); - test_mm_cmp_ps::(); - test_mm_cmp_sd::(); - test_mm_cmp_pd::(); - test_mm256_cmp_ps::(); - test_mm256_cmp_pd::(); - } - - test_cmp::<_CMP_EQ_OQ>(); - test_cmp::<_CMP_LT_OS>(); - test_cmp::<_CMP_LE_OS>(); - test_cmp::<_CMP_UNORD_Q>(); - test_cmp::<_CMP_NEQ_UQ>(); - test_cmp::<_CMP_NLT_UQ>(); - test_cmp::<_CMP_NLE_UQ>(); - test_cmp::<_CMP_ORD_Q>(); - test_cmp::<_CMP_EQ_UQ>(); - test_cmp::<_CMP_NGE_US>(); - test_cmp::<_CMP_NGT_US>(); - test_cmp::<_CMP_FALSE_OQ>(); - test_cmp::<_CMP_NEQ_OQ>(); - test_cmp::<_CMP_GE_OS>(); - test_cmp::<_CMP_GT_OS>(); - test_cmp::<_CMP_TRUE_US>(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cvtps_epi32() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let r = _mm256_cvtps_epi32(a); - let e = _mm256_setr_epi32(4, 9, 16, 25, 4, 9, 16, 25); - assert_eq_m256i(r, e); - - let a = _mm256_setr_ps( - f32::NEG_INFINITY, - f32::INFINITY, - f32::MIN, - f32::MAX, - f32::NAN, - f32::NAN, - f32::NAN, - f32::NAN, - ); - let r = _mm256_cvtps_epi32(a); - assert_eq_m256i(r, _mm256_set1_epi32(i32::MIN)); - } - test_mm256_cvtps_epi32(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cvttps_epi32() { - let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); - let r = _mm256_cvttps_epi32(a); - let e = _mm256_setr_epi32(4, 9, 16, 25, 4, 9, 16, 25); - assert_eq_m256i(r, e); - - let a = _mm256_setr_ps( - f32::NEG_INFINITY, - f32::INFINITY, - f32::MIN, - f32::MAX, - f32::NAN, - f32::NAN, - f32::NAN, - f32::NAN, - ); - let r = _mm256_cvttps_epi32(a); - assert_eq_m256i(r, _mm256_set1_epi32(i32::MIN)); - } - test_mm256_cvttps_epi32(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cvtpd_epi32() { - let a = _mm256_setr_pd(4., 9., 16., 25.); - let r = _mm256_cvtpd_epi32(a); - let e = _mm_setr_epi32(4, 9, 16, 25); - assert_eq_m128i(r, e); - - let a = _mm256_setr_pd(f64::NEG_INFINITY, f64::INFINITY, f64::MIN, f64::MAX); - let r = _mm256_cvtpd_epi32(a); - assert_eq_m128i(r, _mm_set1_epi32(i32::MIN)); - } - test_mm256_cvtpd_epi32(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_cvttpd_epi32() { - let a = _mm256_setr_pd(4., 9., 16., 25.); - let r = _mm256_cvttpd_epi32(a); - let e = _mm_setr_epi32(4, 9, 16, 25); - assert_eq_m128i(r, e); - - let a = _mm256_setr_pd(f64::NEG_INFINITY, f64::INFINITY, f64::MIN, f64::MAX); - let r = _mm256_cvttpd_epi32(a); - assert_eq_m128i(r, _mm_set1_epi32(i32::MIN)); - } - test_mm256_cvttpd_epi32(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_permutevar_ps() { - let a = _mm_setr_ps(4., 3., 2., 5.); - let b = _mm_setr_epi32(1, 2, 3, 4); - let r = _mm_permutevar_ps(a, b); - let e = _mm_setr_ps(3., 2., 5., 4.); - assert_eq_m128(r, e); - } - test_mm_permutevar_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_permutevar_ps() { - let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); - let b = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let r = _mm256_permutevar_ps(a, b); - let e = _mm256_setr_ps(3., 2., 5., 4., 9., 64., 50., 8.); - assert_eq_m256(r, e); - } - test_mm256_permutevar_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_permutevar_pd() { - let a = _mm_setr_pd(4., 3.); - let b = _mm_setr_epi64x(3, 0); - let r = _mm_permutevar_pd(a, b); - let e = _mm_setr_pd(3., 4.); - assert_eq_m128d(r, e); - } - test_mm_permutevar_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_permutevar_pd() { - let a = _mm256_setr_pd(4., 3., 2., 5.); - let b = _mm256_setr_epi64x(1, 2, 3, 4); - let r = _mm256_permutevar_pd(a, b); - let e = _mm256_setr_pd(4., 3., 5., 2.); - assert_eq_m256d(r, e); - } - test_mm256_permutevar_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_permute2f128_ps() { - let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); - let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); - let r = _mm256_permute2f128_ps::<0x13>(a, b); - let e = _mm256_setr_ps(5., 6., 7., 8., 1., 2., 3., 4.); - assert_eq_m256(r, e); - - let r = _mm256_permute2f128_ps::<0x44>(a, b); - let e = _mm256_setr_ps(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); - assert_eq_m256(r, e); - } - test_mm256_permute2f128_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_permute2f128_pd() { - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_permute2f128_pd::<0x31>(a, b); - let e = _mm256_setr_pd(3., 4., 7., 8.); - assert_eq_m256d(r, e); - - let r = _mm256_permute2f128_pd::<0x44>(a, b); - let e = _mm256_setr_pd(0.0, 0.0, 0.0, 0.0); - assert_eq_m256d(r, e); - } - test_mm256_permute2f128_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_permute2f128_si256() { - let a = _mm256_setr_epi32(1, 2, 3, 4, 1, 2, 3, 4); - let b = _mm256_setr_epi32(5, 6, 7, 8, 5, 6, 7, 8); - let r = _mm256_permute2f128_si256::<0x20>(a, b); - let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - assert_eq_m256i(r, e); - - let r = _mm256_permute2f128_si256::<0x44>(a, b); - let e = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, 0); - assert_eq_m256i(r, e); - } - test_mm256_permute2f128_si256(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_maskload_ps() { - let a = &[1.0f32, 2., 3., 4.]; - let mask = _mm_setr_epi32(0, !0, 0, !0); - let r = _mm_maskload_ps(a.as_ptr(), mask); - let e = _mm_setr_ps(0., 2., 0., 4.); - assert_eq_m128(r, e); - - // Unaligned pointer - let a = Unaligned::new([1.0f32, 2., 3., 4.]); - let mask = _mm_setr_epi32(0, !0, 0, !0); - let r = _mm_maskload_ps(a.as_ptr().cast(), mask); - let e = _mm_setr_ps(0., 2., 0., 4.); - assert_eq_m128(r, e); - - // Only loading first element, so slice can be short. - let a = &[2.0f32]; - let mask = _mm_setr_epi32(!0, 0, 0, 0); - let r = _mm_maskload_ps(a.as_ptr(), mask); - let e = _mm_setr_ps(2.0, 0.0, 0.0, 0.0); - assert_eq_m128(r, e); - - // Only loading last element, so slice can be short. - let a = &[2.0f32]; - let mask = _mm_setr_epi32(0, 0, 0, !0); - let r = _mm_maskload_ps(a.as_ptr().wrapping_sub(3), mask); - let e = _mm_setr_ps(0.0, 0.0, 0.0, 2.0); - assert_eq_m128(r, e); - } - test_mm_maskload_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_maskload_pd() { - let a = &[1.0f64, 2.]; - let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_pd(a.as_ptr(), mask); - let e = _mm_setr_pd(0., 2.); - assert_eq_m128d(r, e); - - // Unaligned pointer - let a = Unaligned::new([1.0f64, 2.]); - let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_pd(a.as_ptr().cast(), mask); - let e = _mm_setr_pd(0., 2.); - assert_eq_m128d(r, e); - - // Only loading first element, so slice can be short. - let a = &[2.0f64]; - let mask = _mm_setr_epi64x(!0, 0); - let r = _mm_maskload_pd(a.as_ptr(), mask); - let e = _mm_setr_pd(2.0, 0.0); - assert_eq_m128d(r, e); - - // Only loading last element, so slice can be short. - let a = &[2.0f64]; - let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_pd(a.as_ptr().wrapping_sub(1), mask); - let e = _mm_setr_pd(0.0, 2.0); - assert_eq_m128d(r, e); - } - test_mm_maskload_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_maskload_ps() { - let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let r = _mm256_maskload_ps(a.as_ptr(), mask); - let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); - assert_eq_m256(r, e); - - // Unaligned pointer - let a = Unaligned::new([1.0f32, 2., 3., 4., 5., 6., 7., 8.]); - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let r = _mm256_maskload_ps(a.as_ptr().cast(), mask); - let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); - assert_eq_m256(r, e); - - // Only loading first element, so slice can be short. - let a = &[2.0f32]; - let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_maskload_ps(a.as_ptr(), mask); - let e = _mm256_setr_ps(2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); - assert_eq_m256(r, e); - - // Only loading last element, so slice can be short. - let a = &[2.0f32]; - let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); - let r = _mm256_maskload_ps(a.as_ptr().wrapping_sub(7), mask); - let e = _mm256_setr_ps(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0); - assert_eq_m256(r, e); - } - test_mm256_maskload_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_maskload_pd() { - let a = &[1.0f64, 2., 3., 4.]; - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let r = _mm256_maskload_pd(a.as_ptr(), mask); - let e = _mm256_setr_pd(0., 2., 0., 4.); - assert_eq_m256d(r, e); - - // Unaligned pointer - let a = Unaligned::new([1.0f64, 2., 3., 4.]); - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let r = _mm256_maskload_pd(a.as_ptr().cast(), mask); - let e = _mm256_setr_pd(0., 2., 0., 4.); - assert_eq_m256d(r, e); - - // Only loading first element, so slice can be short. - let a = &[2.0f64]; - let mask = _mm256_setr_epi64x(!0, 0, 0, 0); - let r = _mm256_maskload_pd(a.as_ptr(), mask); - let e = _mm256_setr_pd(2.0, 0.0, 0.0, 0.0); - assert_eq_m256d(r, e); - - // Only loading last element, so slice can be short. - let a = &[2.0f64]; - let mask = _mm256_setr_epi64x(0, 0, 0, !0); - let r = _mm256_maskload_pd(a.as_ptr().wrapping_sub(3), mask); - let e = _mm256_setr_pd(0.0, 0.0, 0.0, 2.0); - assert_eq_m256d(r, e); - } - test_mm256_maskload_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_maskstore_ps() { - let mut r = _mm_set1_ps(0.); - let mask = _mm_setr_epi32(0, !0, 0, !0); - let a = _mm_setr_ps(1., 2., 3., 4.); - _mm_maskstore_ps(&mut r as *mut _ as *mut f32, mask, a); - let e = _mm_setr_ps(0., 2., 0., 4.); - assert_eq_m128(r, e); - - // Unaligned pointer - let mut r = Unaligned::new([0.0f32; 4]); - let mask = _mm_setr_epi32(0, !0, 0, !0); - let a = _mm_setr_ps(1., 2., 3., 4.); - _mm_maskstore_ps(r.as_mut_ptr().cast(), mask, a); - let e = [0., 2., 0., 4.]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0.0f32]; - let mask = _mm_setr_epi32(!0, 0, 0, 0); - let a = _mm_setr_ps(1., 2., 3., 4.); - _mm_maskstore_ps(r.as_mut_ptr(), mask, a); - let e = [1.0f32]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0.0f32]; - let mask = _mm_setr_epi32(0, 0, 0, !0); - let a = _mm_setr_ps(1., 2., 3., 4.); - _mm_maskstore_ps(r.as_mut_ptr().wrapping_sub(3), mask, a); - let e = [4.0f32]; - assert_eq!(r, e); - } - test_mm_maskstore_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_maskstore_pd() { - let mut r = _mm_set1_pd(0.); - let mask = _mm_setr_epi64x(0, !0); - let a = _mm_setr_pd(1., 2.); - _mm_maskstore_pd(&mut r as *mut _ as *mut f64, mask, a); - let e = _mm_setr_pd(0., 2.); - assert_eq_m128d(r, e); - - // Unaligned pointer - let mut r = Unaligned::new([0.0f64; 2]); - let mask = _mm_setr_epi64x(0, !0); - let a = _mm_setr_pd(1., 2.); - _mm_maskstore_pd(r.as_mut_ptr().cast(), mask, a); - let e = [0., 2.]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0.0f64]; - let mask = _mm_setr_epi64x(!0, 0); - let a = _mm_setr_pd(1., 2.); - _mm_maskstore_pd(r.as_mut_ptr(), mask, a); - let e = [1.0f64]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0.0f64]; - let mask = _mm_setr_epi64x(0, !0); - let a = _mm_setr_pd(1., 2.); - _mm_maskstore_pd(r.as_mut_ptr().wrapping_sub(1), mask, a); - let e = [2.0f64]; - assert_eq!(r, e); - } - test_mm_maskstore_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_maskstore_ps() { - let mut r = _mm256_set1_ps(0.); - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - _mm256_maskstore_ps(&mut r as *mut _ as *mut f32, mask, a); - let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); - assert_eq_m256(r, e); - - // Unaligned pointer - let mut r = Unaligned::new([0.0f32; 8]); - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - _mm256_maskstore_ps(r.as_mut_ptr().cast(), mask, a); - let e = [0., 2., 0., 4., 0., 6., 0., 8.]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0.0f32]; - let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - _mm256_maskstore_ps(r.as_mut_ptr(), mask, a); - let e = [1.0f32]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0.0f32]; - let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - _mm256_maskstore_ps(r.as_mut_ptr().wrapping_sub(7), mask, a); - let e = [8.0f32]; - assert_eq!(r, e); - } - test_mm256_maskstore_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_maskstore_pd() { - let mut r = _mm256_set1_pd(0.); - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let a = _mm256_setr_pd(1., 2., 3., 4.); - _mm256_maskstore_pd(&mut r as *mut _ as *mut f64, mask, a); - let e = _mm256_setr_pd(0., 2., 0., 4.); - assert_eq_m256d(r, e); - - // Unaligned pointer - let mut r = Unaligned::new([0.0f64; 4]); - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let a = _mm256_setr_pd(1., 2., 3., 4.); - _mm256_maskstore_pd(r.as_mut_ptr().cast(), mask, a); - let e = [0., 2., 0., 4.]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0.0f64]; - let mask = _mm256_setr_epi64x(!0, 0, 0, 0); - let a = _mm256_setr_pd(1., 2., 3., 4.); - _mm256_maskstore_pd(r.as_mut_ptr(), mask, a); - let e = [1.0f64]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0.0f64]; - let mask = _mm256_setr_epi64x(0, 0, 0, !0); - let a = _mm256_setr_pd(1., 2., 3., 4.); - _mm256_maskstore_pd(r.as_mut_ptr().wrapping_sub(3), mask, a); - let e = [4.0f64]; - assert_eq!(r, e); - } - test_mm256_maskstore_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_lddqu_si256() { - #[rustfmt::skip] - let a = _mm256_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, - ); - let p = &a as *const _; - let r = _mm256_lddqu_si256(p); - #[rustfmt::skip] - let e = _mm256_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, - ); - assert_eq_m256i(r, e); - } - test_mm256_lddqu_si256(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testz_si256() { - let a = _mm256_setr_epi64x(1, 2, 3, 4); - let b = _mm256_setr_epi64x(5, 6, 7, 8); - let r = _mm256_testz_si256(a, b); - assert_eq!(r, 0); - let b = _mm256_set1_epi64x(0); - let r = _mm256_testz_si256(a, b); - assert_eq!(r, 1); - } - test_mm256_testz_si256(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testc_si256() { - let a = _mm256_setr_epi64x(1, 2, 3, 4); - let b = _mm256_setr_epi64x(5, 6, 7, 8); - let r = _mm256_testc_si256(a, b); - assert_eq!(r, 0); - let b = _mm256_set1_epi64x(0); - let r = _mm256_testc_si256(a, b); - assert_eq!(r, 1); - } - test_mm256_testc_si256(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testnzc_si256() { - let a = _mm256_setr_epi64x(1, 2, 3, 4); - let b = _mm256_setr_epi64x(5, 6, 7, 8); - let r = _mm256_testnzc_si256(a, b); - assert_eq!(r, 1); - let a = _mm256_setr_epi64x(0, 0, 0, 0); - let b = _mm256_setr_epi64x(0, 0, 0, 0); - let r = _mm256_testnzc_si256(a, b); - assert_eq!(r, 0); - } - test_mm256_testnzc_si256(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testz_pd() { - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_testz_pd(a, b); - assert_eq!(r, 1); - let a = _mm256_set1_pd(-1.); - let r = _mm256_testz_pd(a, a); - assert_eq!(r, 0); - } - test_mm256_testz_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testc_pd() { - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_testc_pd(a, b); - assert_eq!(r, 1); - let a = _mm256_set1_pd(1.); - let b = _mm256_set1_pd(-1.); - let r = _mm256_testc_pd(a, b); - assert_eq!(r, 0); - } - test_mm256_testc_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testnzc_pd() { - let a = _mm256_setr_pd(1., 2., 3., 4.); - let b = _mm256_setr_pd(5., 6., 7., 8.); - let r = _mm256_testnzc_pd(a, b); - assert_eq!(r, 0); - let a = _mm256_setr_pd(1., -1., -1., -1.); - let b = _mm256_setr_pd(-1., -1., 1., 1.); - let r = _mm256_testnzc_pd(a, b); - assert_eq!(r, 1); - } - test_mm256_testnzc_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testz_pd() { - let a = _mm_setr_pd(1., 2.); - let b = _mm_setr_pd(5., 6.); - let r = _mm_testz_pd(a, b); - assert_eq!(r, 1); - let a = _mm_set1_pd(-1.); - let r = _mm_testz_pd(a, a); - assert_eq!(r, 0); - } - test_mm_testz_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testc_pd() { - let a = _mm_setr_pd(1., 2.); - let b = _mm_setr_pd(5., 6.); - let r = _mm_testc_pd(a, b); - assert_eq!(r, 1); - let a = _mm_set1_pd(1.); - let b = _mm_set1_pd(-1.); - let r = _mm_testc_pd(a, b); - assert_eq!(r, 0); - } - test_mm_testc_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testnzc_pd() { - let a = _mm_setr_pd(1., 2.); - let b = _mm_setr_pd(5., 6.); - let r = _mm_testnzc_pd(a, b); - assert_eq!(r, 0); - let a = _mm_setr_pd(1., -1.); - let b = _mm_setr_pd(-1., -1.); - let r = _mm_testnzc_pd(a, b); - assert_eq!(r, 1); - } - test_mm_testnzc_pd(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testz_ps() { - let a = _mm256_set1_ps(1.); - let r = _mm256_testz_ps(a, a); - assert_eq!(r, 1); - let a = _mm256_set1_ps(-1.); - let r = _mm256_testz_ps(a, a); - assert_eq!(r, 0); - } - test_mm256_testz_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testc_ps() { - let a = _mm256_set1_ps(1.); - let r = _mm256_testc_ps(a, a); - assert_eq!(r, 1); - let b = _mm256_set1_ps(-1.); - let r = _mm256_testc_ps(a, b); - assert_eq!(r, 0); - } - test_mm256_testc_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm256_testnzc_ps() { - let a = _mm256_set1_ps(1.); - let r = _mm256_testnzc_ps(a, a); - assert_eq!(r, 0); - let a = _mm256_setr_ps(1., -1., -1., -1., -1., -1., -1., -1.); - let b = _mm256_setr_ps(-1., -1., 1., 1., 1., 1., 1., 1.); - let r = _mm256_testnzc_ps(a, b); - assert_eq!(r, 1); - } - test_mm256_testnzc_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testz_ps() { - let a = _mm_set1_ps(1.); - let r = _mm_testz_ps(a, a); - assert_eq!(r, 1); - let a = _mm_set1_ps(-1.); - let r = _mm_testz_ps(a, a); - assert_eq!(r, 0); - } - test_mm_testz_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testc_ps() { - let a = _mm_set1_ps(1.); - let r = _mm_testc_ps(a, a); - assert_eq!(r, 1); - let b = _mm_set1_ps(-1.); - let r = _mm_testc_ps(a, b); - assert_eq!(r, 0); - } - test_mm_testc_ps(); - - #[target_feature(enable = "avx")] - unsafe fn test_mm_testnzc_ps() { - let a = _mm_set1_ps(1.); - let r = _mm_testnzc_ps(a, a); - assert_eq!(r, 0); - let a = _mm_setr_ps(1., -1., -1., -1.); - let b = _mm_setr_ps(-1., -1., 1., 1.); - let r = _mm_testnzc_ps(a, b); - assert_eq!(r, 1); - } - test_mm_testnzc_ps(); -} - -#[target_feature(enable = "sse2")] -unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { - _mm_set_epi64x(b, a) -} - -#[track_caller] -#[target_feature(enable = "sse")] -unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { - if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256(a: __m256, b: __m256) { - let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); - if _mm256_movemask_ps(cmp) != 0b11111111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { - let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); - if _mm256_movemask_pd(cmp) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { - assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) -} - -/// Stores `T` in an unaligned address -struct Unaligned { - buf: Vec, - offset: bool, - _marker: std::marker::PhantomData, -} - -impl Unaligned { - fn new(value: T) -> Self { - // Allocate extra byte for unalignment headroom - let len = std::mem::size_of::(); - let mut buf = Vec::::with_capacity(len + 1); - // Force the address to be a non-multiple of 2, so it is as unaligned as it can get. - let offset = (buf.as_ptr() as usize % 2) == 0; - let value_ptr: *const T = &value; - unsafe { - buf.as_mut_ptr().add(offset.into()).copy_from_nonoverlapping(value_ptr.cast(), len); - } - Self { buf, offset, _marker: std::marker::PhantomData } - } - - fn as_ptr(&self) -> *const T { - unsafe { self.buf.as_ptr().add(self.offset.into()).cast() } - } - - fn as_mut_ptr(&mut self) -> *mut T { - unsafe { self.buf.as_mut_ptr().add(self.offset.into()).cast() } - } - - fn read(&self) -> T { - unsafe { self.as_ptr().read_unaligned() } - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx2.rs b/src/tools/miri/tests/pass/intrinsics-x86-avx2.rs deleted file mode 100644 index 80d125bb856..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-avx2.rs +++ /dev/null @@ -1,1613 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=+avx2 - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; -use std::mem::transmute; - -fn main() { - assert!(is_x86_feature_detected!("avx2")); - - unsafe { - test_avx2(); - } -} - -#[target_feature(enable = "avx2")] -unsafe fn test_avx2() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx2.rs - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_abs_epi32() { - #[rustfmt::skip] - let a = _mm256_setr_epi32( - 0, 1, -1, i32::MAX, - i32::MIN, 100, -100, -32, - ); - let r = _mm256_abs_epi32(a); - #[rustfmt::skip] - let e = _mm256_setr_epi32( - 0, 1, 1, i32::MAX, - i32::MAX.wrapping_add(1), 100, 100, 32, - ); - assert_eq_m256i(r, e); - } - test_mm256_abs_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_abs_epi16() { - #[rustfmt::skip] - let a = _mm256_setr_epi16( - 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, i16::MAX, i16::MIN, 100, -100, -32, - ); - let r = _mm256_abs_epi16(a); - #[rustfmt::skip] - let e = _mm256_setr_epi16( - 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, i16::MAX, i16::MAX.wrapping_add(1), 100, 100, 32, - ); - assert_eq_m256i(r, e); - } - test_mm256_abs_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_abs_epi8() { - #[rustfmt::skip] - let a = _mm256_setr_epi8( - 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, - 0, 1, -1, 2, -2, 3, -3, 4, - -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, - ); - let r = _mm256_abs_epi8(a); - #[rustfmt::skip] - let e = _mm256_setr_epi8( - 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, - 0, 1, 1, 2, 2, 3, 3, 4, - 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, - ); - assert_eq_m256i(r, e); - } - test_mm256_abs_epi8(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hadd_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_hadd_epi16(a, b); - let e = _mm256_setr_epi16(4, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8); - assert_eq_m256i(r, e); - - // Test wrapping on overflow - let a = _mm256_setr_epi16( - i16::MAX, - 1, - i16::MAX, - 2, - i16::MAX, - 3, - i16::MAX, - 4, - i16::MAX, - 5, - i16::MAX, - 6, - i16::MAX, - 7, - i16::MAX, - 8, - ); - let b = _mm256_setr_epi16( - i16::MIN, - -1, - i16::MIN, - -2, - i16::MIN, - -3, - i16::MIN, - -4, - i16::MIN, - -5, - i16::MIN, - -6, - i16::MIN, - -7, - i16::MIN, - -8, - ); - let expected = _mm256_setr_epi16( - i16::MIN, - i16::MIN + 1, - i16::MIN + 2, - i16::MIN + 3, - i16::MAX, - i16::MAX - 1, - i16::MAX - 2, - i16::MAX - 3, - i16::MIN + 4, - i16::MIN + 5, - i16::MIN + 6, - i16::MIN + 7, - i16::MAX - 4, - i16::MAX - 5, - i16::MAX - 6, - i16::MAX - 7, - ); - let r = _mm256_hadd_epi16(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hadd_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hadd_epi32() { - let a = _mm256_set1_epi32(2); - let b = _mm256_set1_epi32(4); - let r = _mm256_hadd_epi32(a, b); - let e = _mm256_setr_epi32(4, 4, 8, 8, 4, 4, 8, 8); - assert_eq_m256i(r, e); - - // Test wrapping on overflow - let a = _mm256_setr_epi32(i32::MAX, 1, i32::MAX, 2, i32::MAX, 3, i32::MAX, 4); - let b = _mm256_setr_epi32(i32::MIN, -1, i32::MIN, -2, i32::MIN, -3, i32::MIN, -4); - let expected = _mm256_setr_epi32( - i32::MIN, - i32::MIN + 1, - i32::MAX, - i32::MAX - 1, - i32::MIN + 2, - i32::MIN + 3, - i32::MAX - 2, - i32::MAX - 3, - ); - let r = _mm256_hadd_epi32(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hadd_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hadds_epi16() { - let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16::<0>(a, 0x7fff); - let a = _mm256_insert_epi16::<1>(a, 1); - let b = _mm256_set1_epi16(4); - let r = _mm256_hadds_epi16(a, b); - let e = _mm256_setr_epi16(0x7FFF, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8); - assert_eq_m256i(r, e); - - // Test saturating on overflow - let a = _mm256_setr_epi16( - i16::MAX, - 1, - i16::MAX, - 2, - i16::MAX, - 3, - i16::MAX, - 4, - i16::MAX, - 5, - i16::MAX, - 6, - i16::MAX, - 7, - i16::MAX, - 8, - ); - let b = _mm256_setr_epi16( - i16::MIN, - -1, - i16::MIN, - -2, - i16::MIN, - -3, - i16::MIN, - -4, - i16::MIN, - -5, - i16::MIN, - -6, - i16::MIN, - -7, - i16::MIN, - -8, - ); - let expected = _mm256_setr_epi16( - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - ); - let r = _mm256_hadds_epi16(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hadds_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hsub_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_hsub_epi16(a, b); - let e = _mm256_set1_epi16(0); - assert_eq_m256i(r, e); - - // Test wrapping on overflow - let a = _mm256_setr_epi16( - i16::MAX, - -1, - i16::MAX, - -2, - i16::MAX, - -3, - i16::MAX, - -4, - i16::MAX, - -5, - i16::MAX, - -6, - i16::MAX, - -7, - i16::MAX, - -8, - ); - let b = _mm256_setr_epi16( - i16::MIN, - 1, - i16::MIN, - 2, - i16::MIN, - 3, - i16::MIN, - 4, - i16::MIN, - 5, - i16::MIN, - 6, - i16::MIN, - 7, - i16::MIN, - 8, - ); - let expected = _mm256_setr_epi16( - i16::MIN, - i16::MIN + 1, - i16::MIN + 2, - i16::MIN + 3, - i16::MAX, - i16::MAX - 1, - i16::MAX - 2, - i16::MAX - 3, - i16::MIN + 4, - i16::MIN + 5, - i16::MIN + 6, - i16::MIN + 7, - i16::MAX - 4, - i16::MAX - 5, - i16::MAX - 6, - i16::MAX - 7, - ); - let r = _mm256_hsub_epi16(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hsub_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hsub_epi32() { - let a = _mm256_set1_epi32(2); - let b = _mm256_set1_epi32(4); - let r = _mm256_hsub_epi32(a, b); - let e = _mm256_set1_epi32(0); - assert_eq_m256i(r, e); - - // Test wrapping on overflow - let a = _mm256_setr_epi32(i32::MAX, -1, i32::MAX, -2, i32::MAX, -3, i32::MAX, -4); - let b = _mm256_setr_epi32(i32::MIN, 1, i32::MIN, 2, i32::MIN, 3, i32::MIN, 4); - let expected = _mm256_setr_epi32( - i32::MIN, - i32::MIN + 1, - i32::MAX, - i32::MAX - 1, - i32::MIN + 2, - i32::MIN + 3, - i32::MAX - 2, - i32::MAX - 3, - ); - let r = _mm256_hsub_epi32(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hsub_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_hsubs_epi16() { - let a = _mm256_set1_epi16(2); - let a = _mm256_insert_epi16::<0>(a, 0x7fff); - let a = _mm256_insert_epi16::<1>(a, -1); - let b = _mm256_set1_epi16(4); - let r = _mm256_hsubs_epi16(a, b); - let e = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 0x7FFF); - assert_eq_m256i(r, e); - - // Test saturating on overflow - let a = _mm256_setr_epi16( - i16::MAX, - -1, - i16::MAX, - -2, - i16::MAX, - -3, - i16::MAX, - -4, - i16::MAX, - -5, - i16::MAX, - -6, - i16::MAX, - -7, - i16::MAX, - -8, - ); - let b = _mm256_setr_epi16( - i16::MIN, - 1, - i16::MIN, - 2, - i16::MIN, - 3, - i16::MIN, - 4, - i16::MIN, - 5, - i16::MIN, - 6, - i16::MIN, - 7, - i16::MIN, - 8, - ); - let expected = _mm256_setr_epi16( - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - ); - let r = _mm256_hsubs_epi16(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_hsubs_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i32gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); - assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); - } - test_mm_i32gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm_mask_i32gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm_setr_epi32(-1, -1, -1, 0), - ); - assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); - } - test_mm_mask_i32gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i32gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = - _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); - assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); - } - test_mm256_i32gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm256_mask_i32gather_epi32::<4>( - _mm256_set1_epi32(256), - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), - _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), - ); - assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256)); - } - test_mm256_mask_i32gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i32gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); - assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); - } - test_mm_i32gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i32gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - ); - assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); - } - test_mm_mask_i32gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i32gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = - _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); - assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0)); - } - test_mm256_i32gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i32gather_ps::<4>( - _mm256_set1_ps(256.0), - arr.as_ptr(), - _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), - _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), - ); - assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 64.0, 256.0, 256.0, 256.0, 256.0, 256.0)); - } - test_mm256_mask_i32gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i32gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); - assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); - } - test_mm_i32gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i32gather_epi64::<8>( - _mm_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi32(16, 16, 16, 16), - _mm_setr_epi64x(-1, 0), - ); - assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); - } - test_mm_mask_i32gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i32gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); - assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); - } - test_mm256_i32gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i32gather_epi64::<8>( - _mm256_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm256_setr_epi64x(-1, -1, -1, 0), - ); - assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); - } - test_mm256_mask_i32gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i32gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); - assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); - } - test_mm_i32gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i32gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i32gather_pd::<8>( - _mm_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi32(16, 16, 16, 16), - _mm_setr_pd(-1.0, 0.0), - ); - assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); - } - test_mm_mask_i32gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i32gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); - assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); - } - test_mm256_i32gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i32gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i32gather_pd::<8>( - _mm256_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi32(0, 16, 64, 96), - _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - ); - assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); - } - test_mm256_mask_i32gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i64gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); - assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0)); - } - test_mm_i64gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm_mask_i64gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm_setr_epi64x(0, 16), - _mm_setr_epi32(-1, 0, -1, 0), - ); - assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0)); - } - test_mm_mask_i64gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i64gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); - assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); - } - test_mm256_i64gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_epi32() { - let arr: [i32; 128] = core::array::from_fn(|i| i as i32); - // A multiplier of 4 is word-addressing - let r = _mm256_mask_i64gather_epi32::<4>( - _mm_set1_epi32(256), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm_setr_epi32(-1, -1, -1, 0), - ); - assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); - } - test_mm256_mask_i64gather_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i64gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); - assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0)); - } - test_mm_i64gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm_mask_i64gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm_setr_epi64x(0, 16), - _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), - ); - assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0)); - } - test_mm_mask_i64gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i64gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); - assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); - } - test_mm256_i64gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_ps() { - let arr: [f32; 128] = core::array::from_fn(|i| i as f32); - // A multiplier of 4 is word-addressing for f32s - let r = _mm256_mask_i64gather_ps::<4>( - _mm_set1_ps(256.0), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), - ); - assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); - } - test_mm256_mask_i64gather_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i64gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); - assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); - } - test_mm_i64gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm_mask_i64gather_epi64::<8>( - _mm_set1_epi64x(256), - arr.as_ptr(), - _mm_setr_epi64x(16, 16), - _mm_setr_epi64x(-1, 0), - ); - assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); - } - test_mm_mask_i64gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i64gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); - assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); - } - test_mm256_i64gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_epi64() { - let arr: [i64; 128] = core::array::from_fn(|i| i as i64); - // A multiplier of 8 is word-addressing for i64s - let r = _mm256_mask_i64gather_epi64::<8>( - _mm256_set1_epi64x(256), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm256_setr_epi64x(-1, -1, -1, 0), - ); - assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); - } - test_mm256_mask_i64gather_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_i64gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); - assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); - } - test_mm_i64gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_mask_i64gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm_mask_i64gather_pd::<8>( - _mm_set1_pd(256.0), - arr.as_ptr(), - _mm_setr_epi64x(16, 16), - _mm_setr_pd(-1.0, 0.0), - ); - assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); - } - test_mm_mask_i64gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_i64gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); - assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); - } - test_mm256_i64gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mask_i64gather_pd() { - let arr: [f64; 128] = core::array::from_fn(|i| i as f64); - // A multiplier of 8 is word-addressing for f64s - let r = _mm256_mask_i64gather_pd::<8>( - _mm256_set1_pd(256.0), - arr.as_ptr(), - _mm256_setr_epi64x(0, 16, 64, 96), - _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), - ); - assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); - } - test_mm256_mask_i64gather_pd(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_madd_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_madd_epi16(a, b); - let e = _mm256_set1_epi32(16); - assert_eq_m256i(r, e); - } - test_mm256_madd_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_maddubs_epi16() { - let a = _mm256_set1_epi8(2); - let b = _mm256_set1_epi8(4); - let r = _mm256_maddubs_epi16(a, b); - let e = _mm256_set1_epi16(16); - assert_eq_m256i(r, e); - } - test_mm256_maddubs_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_maskload_epi32() { - let nums = [1, 2, 3, 4]; - let a = &nums as *const i32; - let mask = _mm_setr_epi32(-1, 0, 0, -1); - let r = _mm_maskload_epi32(a, mask); - let e = _mm_setr_epi32(1, 0, 0, 4); - assert_eq_m128i(r, e); - - // Unaligned pointer - let a = Unaligned::new([1i32, 2, 3, 4]); - let mask = _mm_setr_epi32(0, !0, 0, !0); - let r = _mm_maskload_epi32(a.as_ptr().cast(), mask); - let e = _mm_setr_epi32(0, 2, 0, 4); - assert_eq_m128i(r, e); - - // Only loading first element, so slice can be short. - let a = &[2i32]; - let mask = _mm_setr_epi32(!0, 0, 0, 0); - let r = _mm_maskload_epi32(a.as_ptr(), mask); - let e = _mm_setr_epi32(2, 0, 0, 0); - assert_eq_m128i(r, e); - - // Only loading last element, so slice can be short. - let a = &[2i32]; - let mask = _mm_setr_epi32(0, 0, 0, !0); - let r = _mm_maskload_epi32(a.as_ptr().wrapping_sub(3), mask); - let e = _mm_setr_epi32(0, 0, 0, 2); - assert_eq_m128i(r, e); - } - test_mm_maskload_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_maskload_epi32() { - let nums = [1, 2, 3, 4, 5, 6, 7, 8]; - let a = &nums as *const i32; - let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); - let r = _mm256_maskload_epi32(a, mask); - let e = _mm256_setr_epi32(1, 0, 0, 4, 0, 6, 7, 0); - assert_eq_m256i(r, e); - - // Unaligned pointer - let a = Unaligned::new([1i32, 2, 3, 4, 5, 6, 7, 8]); - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let r = _mm256_maskload_epi32(a.as_ptr().cast(), mask); - let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 6, 0, 8); - assert_eq_m256i(r, e); - - // Only loading first element, so slice can be short. - let a = &[2i32]; - let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); - let r = _mm256_maskload_epi32(a.as_ptr(), mask); - let e = _mm256_setr_epi32(2, 0, 0, 0, 0, 0, 0, 0); - assert_eq_m256i(r, e); - - // Only loading last element, so slice can be short. - let a = &[2i32]; - let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); - let r = _mm256_maskload_epi32(a.as_ptr().wrapping_sub(7), mask); - let e = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, 2); - assert_eq_m256i(r, e); - } - test_mm256_maskload_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_maskload_epi64() { - let nums = [1_i64, 2_i64]; - let a = &nums as *const i64; - let mask = _mm_setr_epi64x(0, -1); - let r = _mm_maskload_epi64(a, mask); - let e = _mm_setr_epi64x(0, 2); - assert_eq_m128i(r, e); - - // Unaligned pointer - let a = Unaligned::new([1i64, 2]); - let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_epi64(a.as_ptr().cast(), mask); - let e = _mm_setr_epi64x(0, 2); - assert_eq_m128i(r, e); - - // Only loading first element, so slice can be short. - let a = &[2i64]; - let mask = _mm_setr_epi64x(!0, 0); - let r = _mm_maskload_epi64(a.as_ptr(), mask); - let e = _mm_setr_epi64x(2, 0); - assert_eq_m128i(r, e); - - // Only loading last element, so slice can be short. - let a = &[2i64]; - let mask = _mm_setr_epi64x(0, !0); - let r = _mm_maskload_epi64(a.as_ptr().wrapping_sub(1), mask); - let e = _mm_setr_epi64x(0, 2); - assert_eq_m128i(r, e); - } - test_mm_maskload_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_maskload_epi64() { - let nums = [1_i64, 2_i64, 3_i64, 4_i64]; - let a = &nums as *const i64; - let mask = _mm256_setr_epi64x(0, -1, -1, 0); - let r = _mm256_maskload_epi64(a, mask); - let e = _mm256_setr_epi64x(0, 2, 3, 0); - assert_eq_m256i(r, e); - - // Unaligned pointer - let a = Unaligned::new([1i64, 2, 3, 4]); - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let r = _mm256_maskload_epi64(a.as_ptr().cast(), mask); - let e = _mm256_setr_epi64x(0, 2, 0, 4); - assert_eq_m256i(r, e); - - // Only loading first element, so slice can be short. - let a = &[2i64]; - let mask = _mm256_setr_epi64x(!0, 0, 0, 0); - let r = _mm256_maskload_epi64(a.as_ptr(), mask); - let e = _mm256_setr_epi64x(2, 0, 0, 0); - assert_eq_m256i(r, e); - - // Only loading last element, so slice can be short. - let a = &[2i64]; - let mask = _mm256_setr_epi64x(0, 0, 0, !0); - let r = _mm256_maskload_epi64(a.as_ptr().wrapping_sub(3), mask); - let e = _mm256_setr_epi64x(0, 0, 0, 2); - assert_eq_m256i(r, e); - } - test_mm256_maskload_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_maskstore_epi32() { - let a = _mm_setr_epi32(1, 2, 3, 4); - let mut arr = [-1, -1, -1, -1]; - let mask = _mm_setr_epi32(-1, 0, 0, -1); - _mm_maskstore_epi32(arr.as_mut_ptr(), mask, a); - let e = [1, -1, -1, 4]; - assert_eq!(arr, e); - - // Unaligned pointer - let mut r = Unaligned::new([0i32; 4]); - let mask = _mm_setr_epi32(0, !0, 0, !0); - let a = _mm_setr_epi32(1, 2, 3, 4); - _mm_maskstore_epi32(r.as_mut_ptr().cast(), mask, a); - let e = [0i32, 2, 0, 4]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0i32]; - let mask = _mm_setr_epi32(!0, 0, 0, 0); - let a = _mm_setr_epi32(1, 2, 3, 4); - _mm_maskstore_epi32(r.as_mut_ptr(), mask, a); - let e = [1i32]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0i32]; - let mask = _mm_setr_epi32(0, 0, 0, !0); - let a = _mm_setr_epi32(1, 2, 3, 4); - _mm_maskstore_epi32(r.as_mut_ptr().wrapping_sub(3), mask, a); - let e = [4i32]; - assert_eq!(r, e); - } - test_mm_maskstore_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_maskstore_epi32() { - let a = _mm256_setr_epi32(1, 0x6d726f, 3, 42, 0x777161, 6, 7, 8); - let mut arr = [-1, -1, -1, 0x776173, -1, 0x68657265, -1, -1]; - let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); - _mm256_maskstore_epi32(arr.as_mut_ptr(), mask, a); - let e = [1, -1, -1, 42, -1, 6, 7, -1]; - assert_eq!(arr, e); - - // Unaligned pointer - let mut r = Unaligned::new([0i32; 8]); - let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); - let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - _mm256_maskstore_epi32(r.as_mut_ptr().cast(), mask, a); - let e = [0i32, 2, 0, 4, 0, 6, 0, 8]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0i32]; - let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); - let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - _mm256_maskstore_epi32(r.as_mut_ptr(), mask, a); - let e = [1i32]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0i32]; - let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); - let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); - _mm256_maskstore_epi32(r.as_mut_ptr().wrapping_sub(7), mask, a); - let e = [8i32]; - assert_eq!(r, e); - } - test_mm256_maskstore_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_maskstore_epi64() { - let a = _mm_setr_epi64x(1_i64, 2_i64); - let mut arr = [-1_i64, -1_i64]; - let mask = _mm_setr_epi64x(0, -1); - _mm_maskstore_epi64(arr.as_mut_ptr(), mask, a); - let e = [-1, 2]; - assert_eq!(arr, e); - - // Unaligned pointer - let mut r = Unaligned::new([0i64; 2]); - let mask = _mm_setr_epi64x(0, !0); - let a = _mm_setr_epi64x(1, 2); - _mm_maskstore_epi64(r.as_mut_ptr().cast(), mask, a); - let e = [0i64, 2]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0i64]; - let mask = _mm_setr_epi64x(!0, 0); - let a = _mm_setr_epi64x(1, 2); - _mm_maskstore_epi64(r.as_mut_ptr(), mask, a); - let e = [1i64]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0i64]; - let mask = _mm_setr_epi64x(0, !0); - let a = _mm_setr_epi64x(1, 2); - _mm_maskstore_epi64(r.as_mut_ptr().wrapping_sub(1), mask, a); - let e = [2i64]; - assert_eq!(r, e); - } - test_mm_maskstore_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_maskstore_epi64() { - let a = _mm256_setr_epi64x(1_i64, 2_i64, 3_i64, 4_i64); - let mut arr = [-1_i64, -1_i64, -1_i64, -1_i64]; - let mask = _mm256_setr_epi64x(0, -1, -1, 0); - _mm256_maskstore_epi64(arr.as_mut_ptr(), mask, a); - let e = [-1, 2, 3, -1]; - assert_eq!(arr, e); - - // Unaligned pointer - let mut r = Unaligned::new([0i64; 4]); - let mask = _mm256_setr_epi64x(0, !0, 0, !0); - let a = _mm256_setr_epi64x(1, 2, 3, 4); - _mm256_maskstore_epi64(r.as_mut_ptr().cast(), mask, a); - let e = [0i64, 2, 0, 4]; - assert_eq!(r.read(), e); - - // Only storing first element, so slice can be short. - let mut r = [0i64]; - let mask = _mm256_setr_epi64x(!0, 0, 0, 0); - let a = _mm256_setr_epi64x(1, 2, 3, 4); - _mm256_maskstore_epi64(r.as_mut_ptr(), mask, a); - let e = [1i64]; - assert_eq!(r, e); - - // Only storing last element, so slice can be short. - let mut r = [0i64]; - let mask = _mm256_setr_epi64x(0, 0, 0, !0); - let a = _mm256_setr_epi64x(1, 2, 3, 4); - _mm256_maskstore_epi64(r.as_mut_ptr().wrapping_sub(3), mask, a); - let e = [4i64]; - assert_eq!(r, e); - } - test_mm256_maskstore_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mpsadbw_epu8() { - let a = _mm256_setr_epi8( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 2, 4, 6, 8, 10, 12, 14, 16, - 18, 20, 22, 24, 26, 28, 30, - ); - - let r = _mm256_mpsadbw_epu8::<0b000>(a, a); - let e = _mm256_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28, 0, 8, 16, 24, 32, 40, 48, 56); - assert_eq_m256i(r, e); - - let r = _mm256_mpsadbw_epu8::<0b001>(a, a); - let e = _mm256_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12, 32, 24, 16, 8, 0, 8, 16, 24); - assert_eq_m256i(r, e); - - let r = _mm256_mpsadbw_epu8::<0b100>(a, a); - let e = _mm256_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44, 32, 40, 48, 56, 64, 72, 80, 88); - assert_eq_m256i(r, e); - - let r = _mm256_mpsadbw_epu8::<0b101>(a, a); - let e = _mm256_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28, 0, 8, 16, 24, 32, 40, 48, 56); - assert_eq_m256i(r, e); - - let r = _mm256_mpsadbw_epu8::<0b111>(a, a); - let e = _mm256_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4, 64, 56, 48, 40, 32, 24, 16, 8); - assert_eq_m256i(r, e); - } - test_mm256_mpsadbw_epu8(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_mulhrs_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_mullo_epi16(a, b); - let e = _mm256_set1_epi16(8); - assert_eq_m256i(r, e); - } - test_mm256_mulhrs_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_packs_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_packs_epi16(a, b); - #[rustfmt::skip] - let e = _mm256_setr_epi8( - 2, 2, 2, 2, 2, 2, 2, 2, - 4, 4, 4, 4, 4, 4, 4, 4, - 2, 2, 2, 2, 2, 2, 2, 2, - 4, 4, 4, 4, 4, 4, 4, 4, - ); - - assert_eq_m256i(r, e); - } - test_mm256_packs_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_packs_epi32() { - let a = _mm256_set1_epi32(2); - let b = _mm256_set1_epi32(4); - let r = _mm256_packs_epi32(a, b); - let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4); - - assert_eq_m256i(r, e); - } - test_mm256_packs_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_packus_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(4); - let r = _mm256_packus_epi16(a, b); - #[rustfmt::skip] - let e = _mm256_setr_epi8( - 2, 2, 2, 2, 2, 2, 2, 2, - 4, 4, 4, 4, 4, 4, 4, 4, - 2, 2, 2, 2, 2, 2, 2, 2, - 4, 4, 4, 4, 4, 4, 4, 4, - ); - - assert_eq_m256i(r, e); - } - test_mm256_packus_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_packus_epi32() { - let a = _mm256_set1_epi32(2); - let b = _mm256_set1_epi32(4); - let r = _mm256_packus_epi32(a, b); - let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4); - - assert_eq_m256i(r, e); - } - test_mm256_packus_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_permutevar8x32_epi32() { - let a = _mm256_setr_epi32(100, 200, 300, 400, 500, 600, 700, 800); - let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4); - let expected = _mm256_setr_epi32(600, 100, 600, 200, 800, 700, 400, 500); - let r = _mm256_permutevar8x32_epi32(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_permutevar8x32_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_permute2x128_si256() { - let a = _mm256_setr_epi64x(100, 200, 500, 600); - let b = _mm256_setr_epi64x(300, 400, 700, 800); - let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b); - let e = _mm256_setr_epi64x(700, 800, 500, 600); - assert_eq_m256i(r, e); - } - test_mm256_permute2x128_si256(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_permutevar8x32_ps() { - let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); - let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4); - let r = _mm256_permutevar8x32_ps(a, b); - let e = _mm256_setr_ps(6., 1., 6., 2., 8., 7., 4., 5.); - assert_eq_m256(r, e); - } - test_mm256_permutevar8x32_ps(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sad_epu8() { - let a = _mm256_set1_epi8(2); - let b = _mm256_set1_epi8(4); - let r = _mm256_sad_epu8(a, b); - let e = _mm256_set1_epi64x(16); - assert_eq_m256i(r, e); - } - test_mm256_sad_epu8(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_shuffle_epi8() { - #[rustfmt::skip] - let a = _mm256_setr_epi8( - 1, 2, 3, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, - ); - #[rustfmt::skip] - let b = _mm256_setr_epi8( - 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, - 12, 5, 5, 10, 4, 1, 8, 0, - 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, - 12, 5, 5, 10, 4, 1, 8, 0, - ); - #[rustfmt::skip] - let expected = _mm256_setr_epi8( - 5, 0, 5, 4, 9, 13, 7, 4, - 13, 6, 6, 11, 5, 2, 9, 1, - 21, 0, 21, 20, 25, 29, 23, 20, - 29, 22, 22, 27, 21, 18, 25, 17, - ); - let r = _mm256_shuffle_epi8(a, b); - assert_eq_m256i(r, expected); - } - test_mm256_shuffle_epi8(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sign_epi16() { - let a = _mm256_set1_epi16(2); - let b = _mm256_set1_epi16(-1); - let r = _mm256_sign_epi16(a, b); - let e = _mm256_set1_epi16(-2); - assert_eq_m256i(r, e); - } - test_mm256_sign_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sign_epi32() { - let a = _mm256_set1_epi32(2); - let b = _mm256_set1_epi32(-1); - let r = _mm256_sign_epi32(a, b); - let e = _mm256_set1_epi32(-2); - assert_eq_m256i(r, e); - } - test_mm256_sign_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sign_epi8() { - let a = _mm256_set1_epi8(2); - let b = _mm256_set1_epi8(-1); - let r = _mm256_sign_epi8(a, b); - let e = _mm256_set1_epi8(-2); - assert_eq_m256i(r, e); - } - test_mm256_sign_epi8(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sll_epi16() { - let a = _mm256_setr_epi16( - 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, - -0xEE, 0xFF, -0xFF, - ); - let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi16( - 0x880, -0x880, 0x990, -0x990, 0xAA0, -0xAA0, 0xBB0, -0xBB0, 0xCC0, -0xCC0, 0xDD0, - -0xDD0, 0xEE0, -0xEE0, 0xFF0, -0xFF0, - ), - ); - let r = _mm256_sll_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m256i(r, _mm256_set1_epi16(0)); - let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi16(0)); - } - test_mm256_sll_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sll_epi32() { - let a = - _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi32( - 0xCCCC0, -0xCCCC0, 0xDDDD0, -0xDDDD0, 0xEEEE0, -0xEEEE0, 0xFFFF0, -0xFFFF0, - ), - ); - let r = _mm256_sll_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m256i(r, _mm256_set1_epi32(0)); - let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi32(0)); - } - test_mm256_sll_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sll_epi64() { - let a = _mm256_set_epi64x(0xEEEEEEEE, -0xEEEEEEEE, 0xFFFFFFFF, -0xFFFFFFFF); - let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i(r, _mm256_set_epi64x(0xEEEEEEEE0, -0xEEEEEEEE0, 0xFFFFFFFF0, -0xFFFFFFFF0)); - let r = _mm256_sll_epi64(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, 64)); - assert_eq_m256i(r, _mm256_set1_epi64x(0)); - let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi64x(0)); - } - test_mm256_sll_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sra_epi16() { - let a = _mm256_setr_epi16( - 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, - -0xEE, 0xFF, -0xFF, - ); - let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi16( - 0x8, -0x9, 0x9, -0xA, 0xA, -0xB, 0xB, -0xC, 0xC, -0xD, 0xD, -0xE, 0xE, -0xF, 0xF, - -0x10, - ), - ); - let r = _mm256_sra_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m256i( - r, - _mm256_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1), - ); - let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i( - r, - _mm256_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1), - ); - } - test_mm256_sra_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sra_epi32() { - let a = - _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi32(0xCCC, -0xCCD, 0xDDD, -0xDDE, 0xEEE, -0xEEF, 0xFFF, -0x1000), - ); - let r = _mm256_sra_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m256i(r, _mm256_setr_epi32(0, -1, 0, -1, 0, -1, 0, -1)); - let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_setr_epi32(0, -1, 0, -1, 0, -1, 0, -1)); - } - test_mm256_sra_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srl_epi16() { - let a = _mm256_setr_epi16( - 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, - -0xEE, 0xFF, -0xFF, - ); - let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi16( - 0x8, 0xFF7, 0x9, 0xFF6, 0xA, 0xFF5, 0xB, 0xFF4, 0xC, 0xFF3, 0xD, 0xFF2, 0xE, 0xFF1, - 0xF, 0xFF0, - ), - ); - let r = _mm256_srl_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m256i(r, _mm256_set1_epi16(0)); - let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi16(0)); - } - test_mm256_srl_epi16(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srl_epi32() { - let a = - _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_setr_epi32( - 0xCCC, 0xFFFF333, 0xDDD, 0xFFFF222, 0xEEE, 0xFFFF111, 0xFFF, 0xFFFF000, - ), - ); - let r = _mm256_srl_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m256i(r, _mm256_set1_epi32(0)); - let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi32(0)); - } - test_mm256_srl_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srl_epi64() { - let a = _mm256_set_epi64x(0xEEEEEEEE, -0xEEEEEEEE, 0xFFFFFFFF, -0xFFFFFFFF); - let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, 4)); - assert_eq_m256i( - r, - _mm256_set_epi64x(0xEEEEEEE, 0xFFFFFFFF1111111, 0xFFFFFFF, 0xFFFFFFFF0000000), - ); - let r = _mm256_srl_epi64(a, _mm_set_epi64x(4, 0)); - assert_eq_m256i(r, a); - let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, 64)); - assert_eq_m256i(r, _mm256_set1_epi64x(0)); - let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m256i(r, _mm256_set1_epi64x(0)); - } - test_mm256_srl_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_sllv_epi32() { - let a = _mm_set_epi32(1, 2, 3, 4); - let b = _mm_set_epi32(4, 3, 2, 1); - let r = _mm_sllv_epi32(a, b); - let e = _mm_set_epi32(16, 16, 12, 8); - assert_eq_m128i(r, e); - } - test_mm_sllv_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sllv_epi32() { - let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); - let r = _mm256_sllv_epi32(a, b); - let e = _mm256_set_epi32(256, 256, 192, 128, 80, 48, 28, 16); - assert_eq_m256i(r, e); - } - test_mm256_sllv_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_sllv_epi64() { - let a = _mm_set_epi64x(2, 3); - let b = _mm_set_epi64x(1, 2); - let r = _mm_sllv_epi64(a, b); - let e = _mm_set_epi64x(4, 12); - assert_eq_m128i(r, e); - } - test_mm_sllv_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_sllv_epi64() { - let a = _mm256_set_epi64x(1, 2, 3, 4); - let b = _mm256_set_epi64x(4, 3, 2, 1); - let r = _mm256_sllv_epi64(a, b); - let e = _mm256_set_epi64x(16, 16, 12, 8); - assert_eq_m256i(r, e); - } - test_mm256_sllv_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_srav_epi32() { - let a = _mm_set_epi32(16, -32, 64, -128); - let b = _mm_set_epi32(4, 3, 2, 1); - let r = _mm_srav_epi32(a, b); - let e = _mm_set_epi32(1, -4, 16, -64); - assert_eq_m128i(r, e); - } - test_mm_srav_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srav_epi32() { - let a = _mm256_set_epi32(256, -512, 1024, -2048, 4096, -8192, 16384, -32768); - let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); - let r = _mm256_srav_epi32(a, b); - let e = _mm256_set_epi32(1, -4, 16, -64, 256, -1024, 4096, -16384); - assert_eq_m256i(r, e); - } - test_mm256_srav_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_srlv_epi32() { - let a = _mm_set_epi32(16, 32, 64, 128); - let b = _mm_set_epi32(4, 3, 2, 1); - let r = _mm_srlv_epi32(a, b); - let e = _mm_set_epi32(1, 4, 16, 64); - assert_eq_m128i(r, e); - } - test_mm_srlv_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srlv_epi32() { - let a = _mm256_set_epi32(256, 512, 1024, 2048, 4096, 8192, 16384, 32768); - let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); - let r = _mm256_srlv_epi32(a, b); - let e = _mm256_set_epi32(1, 4, 16, 64, 256, 1024, 4096, 16384); - assert_eq_m256i(r, e); - } - test_mm256_srlv_epi32(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm_srlv_epi64() { - let a = _mm_set_epi64x(4, 8); - let b = _mm_set_epi64x(2, 1); - let r = _mm_srlv_epi64(a, b); - let e = _mm_set_epi64x(1, 4); - assert_eq_m128i(r, e); - } - test_mm_srlv_epi64(); - - #[target_feature(enable = "avx2")] - unsafe fn test_mm256_srlv_epi64() { - let a = _mm256_set_epi64x(16, 32, 64, 128); - let b = _mm256_set_epi64x(4, 3, 2, 1); - let r = _mm256_srlv_epi64(a, b); - let e = _mm256_set_epi64x(1, 4, 16, 64); - assert_eq_m256i(r, e); - } - test_mm256_srlv_epi64(); -} - -#[target_feature(enable = "sse2")] -unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { - _mm_set_epi64x(b, a) -} - -#[track_caller] -#[target_feature(enable = "sse")] -unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { - if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256(a: __m256, b: __m256) { - let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); - if _mm256_movemask_ps(cmp) != 0b11111111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { - let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); - if _mm256_movemask_pd(cmp) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "avx")] -unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { - assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) -} - -/// Stores `T` in an unaligned address -struct Unaligned { - buf: Vec, - offset: bool, - _marker: std::marker::PhantomData, -} - -impl Unaligned { - fn new(value: T) -> Self { - // Allocate extra byte for unalignment headroom - let len = std::mem::size_of::(); - let mut buf = Vec::::with_capacity(len + 1); - // Force the address to be a non-multiple of 2, so it is as unaligned as it can get. - let offset = (buf.as_ptr() as usize % 2) == 0; - let value_ptr: *const T = &value; - unsafe { - buf.as_mut_ptr().add(offset.into()).copy_from_nonoverlapping(value_ptr.cast(), len); - } - Self { buf, offset, _marker: std::marker::PhantomData } - } - - fn as_ptr(&self) -> *const T { - unsafe { self.buf.as_ptr().add(self.offset.into()).cast() } - } - - fn as_mut_ptr(&mut self) -> *mut T { - unsafe { self.buf.as_mut_ptr().add(self.offset.into()).cast() } - } - - fn read(&self) -> T { - unsafe { self.as_ptr().read_unaligned() } - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-avx512.rs b/src/tools/miri/tests/pass/intrinsics-x86-avx512.rs deleted file mode 100644 index 66bfcb20f1c..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-avx512.rs +++ /dev/null @@ -1,217 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=+avx512f,+avx512vl,+avx512bitalg,+avx512vpopcntdq - -#![feature(avx512_target_feature)] -#![feature(stdarch_x86_avx512)] - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; -use std::mem::transmute; - -fn main() { - assert!(is_x86_feature_detected!("avx512f")); - assert!(is_x86_feature_detected!("avx512vl")); - assert!(is_x86_feature_detected!("avx512bitalg")); - assert!(is_x86_feature_detected!("avx512vpopcntdq")); - - unsafe { - test_avx512bitalg(); - test_avx512vpopcntdq(); - } -} - -// Some of the constants in the tests below are just bit patterns. They should not -// be interpreted as integers; signedness does not make sense for them, but -// __mXXXi happens to be defined in terms of signed integers. -#[allow(overflowing_literals)] -#[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] -unsafe fn test_avx512bitalg() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx512bitalg.rs - - #[target_feature(enable = "avx512bitalg,avx512f")] - unsafe fn test_mm512_popcnt_epi16() { - let test_data = _mm512_set_epi16( - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, - 0x3F_FF, 0x7F_FF, 0xFF_FF, -1, -100, 255, 256, 2, 4, 8, 16, 32, 64, 128, 256, 512, - 1024, 2048, - ); - let actual_result = _mm512_popcnt_epi16(test_data); - let reference_result = _mm512_set_epi16( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 12, 8, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, - ); - assert_eq_m512i(actual_result, reference_result); - } - test_mm512_popcnt_epi16(); - - #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] - unsafe fn test_mm256_popcnt_epi16() { - let test_data = _mm256_set_epi16( - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, - 0x3F_FF, 0x7F_FF, - ); - let actual_result = _mm256_popcnt_epi16(test_data); - let reference_result = - _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - assert_eq_m256i(actual_result, reference_result); - } - test_mm256_popcnt_epi16(); - - #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] - unsafe fn test_mm_popcnt_epi16() { - let test_data = _mm_set_epi16(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F); - let actual_result = _mm_popcnt_epi16(test_data); - let reference_result = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); - assert_eq_m128i(actual_result, reference_result); - } - test_mm_popcnt_epi16(); - - #[target_feature(enable = "avx512bitalg,avx512f")] - unsafe fn test_mm512_popcnt_epi8() { - let test_data = _mm512_set_epi8( - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, - 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, 183, 154, 84, 56, 227, 189, - 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, 251, 73, 121, 143, 145, 85, 91, 137, 90, - 225, 21, 249, 211, 155, 228, 70, - ); - let actual_result = _mm512_popcnt_epi8(test_data); - let reference_result = _mm512_set_epi8( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, - 2, 4, 4, 6, 4, 3, 3, 5, 6, 3, 3, 5, 6, 4, 4, 4, 3, 3, 6, 7, 3, 5, 5, 3, 4, 5, 3, 4, 4, - 3, 6, 5, 5, 4, 3, - ); - assert_eq_m512i(actual_result, reference_result); - } - test_mm512_popcnt_epi8(); - - #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] - unsafe fn test_mm256_popcnt_epi8() { - let test_data = _mm256_set_epi8( - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, - 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, - ); - let actual_result = _mm256_popcnt_epi8(test_data); - let reference_result = _mm256_set_epi8( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, - 2, 4, 4, - ); - assert_eq_m256i(actual_result, reference_result); - } - test_mm256_popcnt_epi8(); - - #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] - unsafe fn test_mm_popcnt_epi8() { - let test_data = - _mm_set_epi8(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64); - let actual_result = _mm_popcnt_epi8(test_data); - let reference_result = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1); - assert_eq_m128i(actual_result, reference_result); - } - test_mm_popcnt_epi8(); -} - -#[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] -unsafe fn test_avx512vpopcntdq() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs - - #[target_feature(enable = "avx512vpopcntdq,avx512f")] - unsafe fn test_mm512_popcnt_epi32() { - let test_data = _mm512_set_epi32( - 0, - 1, - -1, - 2, - 7, - 0xFF_FE, - 0x7F_FF_FF_FF, - -100, - 0x40_00_00_00, - 103, - 371, - 552, - 432_948, - 818_826_998, - 255, - 256, - ); - let actual_result = _mm512_popcnt_epi32(test_data); - let reference_result = - _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1); - assert_eq_m512i(actual_result, reference_result); - } - test_mm512_popcnt_epi32(); - - #[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] - unsafe fn test_mm256_popcnt_epi32() { - let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); - let actual_result = _mm256_popcnt_epi32(test_data); - let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28); - assert_eq_m256i(actual_result, reference_result); - } - test_mm256_popcnt_epi32(); - - #[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] - unsafe fn test_mm_popcnt_epi32() { - let test_data = _mm_set_epi32(0, 1, -1, -100); - let actual_result = _mm_popcnt_epi32(test_data); - let reference_result = _mm_set_epi32(0, 1, 32, 28); - assert_eq_m128i(actual_result, reference_result); - } - test_mm_popcnt_epi32(); - - #[target_feature(enable = "avx512vpopcntdq,avx512f")] - unsafe fn test_mm512_popcnt_epi64() { - let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); - let actual_result = _mm512_popcnt_epi64(test_data); - let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60); - assert_eq_m512i(actual_result, reference_result); - } - test_mm512_popcnt_epi64(); - - #[target_feature(enable = "avx512vpopcntdq,avx512vl")] - unsafe fn test_mm256_popcnt_epi64() { - let test_data = _mm256_set_epi64x(0, 1, -1, -100); - let actual_result = _mm256_popcnt_epi64(test_data); - let reference_result = _mm256_set_epi64x(0, 1, 64, 60); - assert_eq_m256i(actual_result, reference_result); - } - test_mm256_popcnt_epi64(); - - #[target_feature(enable = "avx512vpopcntdq,avx512vl")] - unsafe fn test_mm_popcnt_epi64() { - let test_data = _mm_set_epi64x(0, 1); - let actual_result = _mm_popcnt_epi64(test_data); - let reference_result = _mm_set_epi64x(0, 1); - assert_eq_m128i(actual_result, reference_result); - let test_data = _mm_set_epi64x(-1, -100); - let actual_result = _mm_popcnt_epi64(test_data); - let reference_result = _mm_set_epi64x(64, 60); - assert_eq_m128i(actual_result, reference_result); - } - test_mm_popcnt_epi64(); -} - -#[track_caller] -unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) { - assert_eq!(transmute::<_, [i32; 16]>(a), transmute::<_, [i32; 16]>(b)) -} - -#[track_caller] -unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { - assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) -} - -#[track_caller] -unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs b/src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs deleted file mode 100644 index c8b92fd5458..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-pause-without-sse2.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=-sse2 - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - -fn main() { - assert!(!is_x86_feature_detected!("sse2")); - - unsafe { - // This is a SSE2 intrinsic, but it behaves as a no-op when SSE2 - // is not available, so it is always safe to call. - _mm_pause(); - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse.rs deleted file mode 100644 index a62a5ee3781..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-sse.rs +++ /dev/null @@ -1,1107 +0,0 @@ -fn main() { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert!(is_x86_feature_detected!("sse")); - - unsafe { - tests::test_sse(); - } - } -} - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod tests { - #[cfg(target_arch = "x86")] - use std::arch::x86::*; - #[cfg(target_arch = "x86_64")] - use std::arch::x86_64::*; - use std::f32::NAN; - use std::mem::transmute; - - macro_rules! assert_approx_eq { - ($a:expr, $b:expr, $eps:expr) => {{ - let (a, b) = (&$a, &$b); - assert!( - (*a - *b).abs() < $eps, - "assertion failed: `(left !== right)` \ - (left: `{:?}`, right: `{:?}`, expect diff: `{:?}`, real diff: `{:?}`)", - *a, - *b, - $eps, - (*a - *b).abs() - ); - }}; - } - - #[target_feature(enable = "sse")] - pub(super) unsafe fn test_sse() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86{,_64}/sse.rs - - #[target_feature(enable = "sse")] - unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } - } - - #[target_feature(enable = "sse")] - unsafe fn test_mm_add_ss() { - let a = _mm_set_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_set_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_add_ss(a, b); - assert_eq_m128(r, _mm_set_ps(-1.0, 5.0, 0.0, -15.0)); - } - test_mm_add_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_sub_ss() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_sub_ss(a, b); - assert_eq_m128(r, _mm_setr_ps(99.0, 5.0, 0.0, -10.0)); - } - test_mm_sub_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_mul_ss() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_mul_ss(a, b); - assert_eq_m128(r, _mm_setr_ps(100.0, 5.0, 0.0, -10.0)); - } - test_mm_mul_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_div_ss() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_div_ss(a, b); - assert_eq_m128(r, _mm_setr_ps(0.01, 5.0, 0.0, -10.0)); - } - test_mm_div_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_sqrt_ss() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_sqrt_ss(a); - let e = _mm_setr_ps(2.0, 13.0, 16.0, 100.0); - assert_eq_m128(r, e); - } - test_mm_sqrt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_sqrt_ps() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_sqrt_ps(a); - let e = _mm_setr_ps(2.0, 3.6055512, 4.0, 10.0); - assert_eq_m128(r, e); - } - test_mm_sqrt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_rcp_ss() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_rcp_ss(a); - let e = _mm_setr_ps(0.24993896, 13.0, 16.0, 100.0); - let rel_err = 0.00048828125; - - let r: [f32; 4] = transmute(r); - let e: [f32; 4] = transmute(e); - assert_approx_eq!(r[0], e[0], 2. * rel_err); - for i in 1..4 { - assert_eq!(r[i], e[i]); - } - } - test_mm_rcp_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_rcp_ps() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_rcp_ps(a); - let e = _mm_setr_ps(0.24993896, 0.0769043, 0.06248474, 0.0099983215); - let rel_err = 0.00048828125; - - let r: [f32; 4] = transmute(r); - let e: [f32; 4] = transmute(e); - for i in 0..4 { - assert_approx_eq!(r[i], e[i], 2. * rel_err); - } - } - test_mm_rcp_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_rsqrt_ss() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_rsqrt_ss(a); - let e = _mm_setr_ps(0.49987793, 13.0, 16.0, 100.0); - let rel_err = 0.00048828125; - - let r: [f32; 4] = transmute(r); - let e: [f32; 4] = transmute(e); - assert_approx_eq!(r[0], e[0], 2. * rel_err); - for i in 1..4 { - assert_eq!(r[i], e[i]); - } - } - test_mm_rsqrt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_rsqrt_ps() { - let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); - let r = _mm_rsqrt_ps(a); - let e = _mm_setr_ps(0.49987793, 0.2772827, 0.24993896, 0.099990845); - let rel_err = 0.00048828125; - - let r: [f32; 4] = transmute(r); - let e: [f32; 4] = transmute(e); - for i in 0..4 { - assert_approx_eq!(r[i], e[i], 2. * rel_err); - } - } - test_mm_rsqrt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_min_ss() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_min_ss(a, b); - assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); - } - test_mm_min_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_min_ps() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_min_ps(a, b); - assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); - - // `_mm_min_ps` can **not** be implemented using the `simd_min` rust intrinsic because - // the semantics of `simd_min` are different to those of `_mm_min_ps` regarding handling - // of `-0.0`. - let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); - let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_min_ps(a, b)); - let r2: [u8; 16] = transmute(_mm_min_ps(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); - assert_eq!(r1, b); - assert_eq!(r2, a); - assert_ne!(a, b); // sanity check that -0.0 is actually present - } - test_mm_min_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_max_ss() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_max_ss(a, b); - assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, -10.0)); - } - test_mm_max_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_max_ps() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_max_ps(a, b); - assert_eq_m128(r, _mm_setr_ps(-1.0, 20.0, 0.0, -5.0)); - - // `_mm_max_ps` can **not** be implemented using the `simd_max` rust intrinsic because - // the semantics of `simd_max` are different to those of `_mm_max_ps` regarding handling - // of `-0.0`. - let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); - let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_max_ps(a, b)); - let r2: [u8; 16] = transmute(_mm_max_ps(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); - assert_eq!(r1, b); - assert_eq!(r2, a); - assert_ne!(a, b); // sanity check that -0.0 is actually present - } - test_mm_max_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpeq_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(-1.0, 5.0, 6.0, 7.0); - let r: [u32; 4] = transmute(_mm_cmpeq_ss(a, b)); - let e: [u32; 4] = transmute(_mm_setr_ps(transmute(0u32), 2.0, 3.0, 4.0)); - assert_eq!(r, e); - - let b2 = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let r2: [u32; 4] = transmute(_mm_cmpeq_ss(a, b2)); - let e2: [u32; 4] = transmute(_mm_setr_ps(transmute(0xffffffffu32), 2.0, 3.0, 4.0)); - assert_eq!(r2, e2); - } - test_mm_cmpeq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmplt_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = 0u32; // a.extract(0) < b.extract(0) - let c1 = 0u32; // a.extract(0) < c.extract(0) - let d1 = !0u32; // a.extract(0) < d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmplt_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmplt_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmplt_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmplt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmple_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = 0u32; // a.extract(0) <= b.extract(0) - let c1 = !0u32; // a.extract(0) <= c.extract(0) - let d1 = !0u32; // a.extract(0) <= d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmple_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmple_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmple_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmple_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpgt_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) > b.extract(0) - let c1 = 0u32; // a.extract(0) > c.extract(0) - let d1 = 0u32; // a.extract(0) > d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpgt_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpgt_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpgt_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpgt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpge_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) >= b.extract(0) - let c1 = !0u32; // a.extract(0) >= c.extract(0) - let d1 = 0u32; // a.extract(0) >= d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpge_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpge_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpge_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpge_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpneq_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) != b.extract(0) - let c1 = 0u32; // a.extract(0) != c.extract(0) - let d1 = !0u32; // a.extract(0) != d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpneq_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpneq_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpneq_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpneq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnlt_ss() { - // TODO: this test is exactly the same as for `_mm_cmpge_ss`, but there - // must be a difference. It may have to do with behavior in the - // presence of NaNs (signaling or quiet). If so, we should add tests - // for those. - - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) >= b.extract(0) - let c1 = !0u32; // a.extract(0) >= c.extract(0) - let d1 = 0u32; // a.extract(0) >= d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpnlt_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpnlt_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpnlt_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpnlt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnle_ss() { - // TODO: this test is exactly the same as for `_mm_cmpgt_ss`, but there - // must be a difference. It may have to do with behavior in the - // presence - // of NaNs (signaling or quiet). If so, we should add tests for those. - - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) > b.extract(0) - let c1 = 0u32; // a.extract(0) > c.extract(0) - let d1 = 0u32; // a.extract(0) > d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpnle_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpnle_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpnle_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpnle_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpngt_ss() { - // TODO: this test is exactly the same as for `_mm_cmple_ss`, but there - // must be a difference. It may have to do with behavior in the - // presence of NaNs (signaling or quiet). If so, we should add tests - // for those. - - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = 0u32; // a.extract(0) <= b.extract(0) - let c1 = !0u32; // a.extract(0) <= c.extract(0) - let d1 = !0u32; // a.extract(0) <= d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpngt_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpngt_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpngt_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpngt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnge_ss() { - // TODO: this test is exactly the same as for `_mm_cmplt_ss`, but there - // must be a difference. It may have to do with behavior in the - // presence of NaNs (signaling or quiet). If so, we should add tests - // for those. - - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = 0u32; // a.extract(0) < b.extract(0) - let c1 = 0u32; // a.extract(0) < c.extract(0) - let d1 = !0u32; // a.extract(0) < d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpnge_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpnge_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpnge_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpnge_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpord_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = !0u32; // a.extract(0) ord b.extract(0) - let c1 = 0u32; // a.extract(0) ord c.extract(0) - let d1 = !0u32; // a.extract(0) ord d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpord_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpord_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpord_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpord_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpunord_ss() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); - let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); - let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); - - let b1 = 0u32; // a.extract(0) unord b.extract(0) - let c1 = !0u32; // a.extract(0) unord c.extract(0) - let d1 = 0u32; // a.extract(0) unord d.extract(0) - - let rb: [u32; 4] = transmute(_mm_cmpunord_ss(a, b)); - let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); - assert_eq!(rb, eb); - - let rc: [u32; 4] = transmute(_mm_cmpunord_ss(a, c)); - let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); - assert_eq!(rc, ec); - - let rd: [u32; 4] = transmute(_mm_cmpunord_ss(a, d)); - let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); - assert_eq!(rd, ed); - } - test_mm_cmpunord_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpeq_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, fls, tru, fls]; - let r: [u32; 4] = transmute(_mm_cmpeq_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpeq_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmplt_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, fls, fls, fls]; - let r: [u32; 4] = transmute(_mm_cmplt_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmplt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmple_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, 4.0); - let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, fls, tru, fls]; - let r: [u32; 4] = transmute(_mm_cmple_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmple_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpgt_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, tru, fls, fls]; - let r: [u32; 4] = transmute(_mm_cmpgt_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpgt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpge_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, tru, tru, fls]; - let r: [u32; 4] = transmute(_mm_cmpge_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpge_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpneq_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, tru, fls, tru]; - let r: [u32; 4] = transmute(_mm_cmpneq_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpneq_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnlt_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, tru, tru, tru]; - let r: [u32; 4] = transmute(_mm_cmpnlt_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpnlt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnle_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, tru, fls, tru]; - let r: [u32; 4] = transmute(_mm_cmpnle_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpnle_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpngt_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, fls, tru, tru]; - let r: [u32; 4] = transmute(_mm_cmpngt_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpngt_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpnge_ps() { - let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); - let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, fls, fls, tru]; - let r: [u32; 4] = transmute(_mm_cmpnge_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpnge_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpord_ps() { - let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); - let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [tru, fls, fls, fls]; - let r: [u32; 4] = transmute(_mm_cmpord_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpord_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cmpunord_ps() { - let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); - let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); - let tru = !0u32; - let fls = 0u32; - - let e = [fls, tru, tru, tru]; - let r: [u32; 4] = transmute(_mm_cmpunord_ps(a, b)); - assert_eq!(r, e); - } - test_mm_cmpunord_ps(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_comieq_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 0, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_comieq_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_comieq_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_comieq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_comilt_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[0i32, 1, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_comilt_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_comilt_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_comilt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_comile_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 1, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_comile_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_comile_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_comile_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_comigt_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 0, 1, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_comige_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_comige_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_comigt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_comineq_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[0i32, 1, 1, 1]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_comineq_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_comineq_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_comineq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomieq_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 0, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomieq_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomieq_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomieq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomilt_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[0i32, 1, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomilt_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomilt_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomilt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomile_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 1, 0, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomile_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomile_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomile_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomigt_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[0i32, 0, 1, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomigt_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomigt_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomigt_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomige_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[1i32, 0, 1, 0]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomige_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomige_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomige_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_ucomineq_ss() { - let aa = &[3.0f32, 12.0, 23.0, NAN]; - let bb = &[3.0f32, 47.5, 1.5, NAN]; - - let ee = &[0i32, 1, 1, 1]; - - for i in 0..4 { - let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); - let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); - - let r = _mm_ucomineq_ss(a, b); - - assert_eq!( - ee[i], r, - "_mm_ucomineq_ss({:?}, {:?}) = {}, expected: {} (i={})", - a, b, r, ee[i], i - ); - } - } - test_mm_ucomineq_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvtss_si32() { - let inputs = &[42.0f32, -3.1, 4.0e10, 4.0e-20, NAN, 2147483500.1]; - let result = &[42i32, -3, i32::MIN, 0, i32::MIN, 2147483520]; - for i in 0..inputs.len() { - let x = _mm_setr_ps(inputs[i], 1.0, 3.0, 4.0); - let e = result[i]; - let r = _mm_cvtss_si32(x); - assert_eq!( - e, r, - "TestCase #{} _mm_cvtss_si32({:?}) = {}, expected: {}", - i, x, r, e - ); - } - } - test_mm_cvtss_si32(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvttss_si32() { - let inputs = &[ - (42.0f32, 42i32), - (-31.4, -31), - (-33.5, -33), - (-34.5, -34), - (10.999, 10), - (-5.99, -5), - (4.0e10, i32::MIN), - (4.0e-10, 0), - (NAN, i32::MIN), - (2147483500.1, 2147483520), - ]; - for i in 0..inputs.len() { - let (xi, e) = inputs[i]; - let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); - let r = _mm_cvttss_si32(x); - assert_eq!( - e, r, - "TestCase #{} _mm_cvttss_si32({:?}) = {}, expected: {}", - i, x, r, e - ); - } - } - test_mm_cvttss_si32(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvtss_f32() { - let a = _mm_setr_ps(312.0134, 5.0, 6.0, 7.0); - assert_eq!(_mm_cvtss_f32(a), 312.0134); - } - test_mm_cvtss_f32(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvtsi32_ss() { - let inputs = &[ - (4555i32, 4555.0f32), - (322223333, 322223330.0), - (-432, -432.0), - (-322223333, -322223330.0), - ]; - - for i in 0..inputs.len() { - let (x, f) = inputs[i]; - let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); - let r = _mm_cvtsi32_ss(a, x); - let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); - assert_eq_m128(e, r); - } - } - test_mm_cvtsi32_ss(); - - // Intrinsic only available on x86_64 - #[cfg(target_arch = "x86_64")] - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvtss_si64() { - let inputs = &[ - (42.0f32, 42i64), - (-31.4, -31), - (-33.5, -34), - (-34.5, -34), - (4.0e10, 40_000_000_000), - (4.0e-10, 0), - (f32::NAN, i64::MIN), - (2147483500.1, 2147483520), - (9.223371e18, 9223370937343148032), - ]; - for i in 0..inputs.len() { - let (xi, e) = inputs[i]; - let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); - let r = _mm_cvtss_si64(x); - assert_eq!( - e, r, - "TestCase #{} _mm_cvtss_si64({:?}) = {}, expected: {}", - i, x, r, e - ); - } - } - #[cfg(target_arch = "x86_64")] - test_mm_cvtss_si64(); - - // Intrinsic only available on x86_64 - #[cfg(target_arch = "x86_64")] - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvttss_si64() { - let inputs = &[ - (42.0f32, 42i64), - (-31.4, -31), - (-33.5, -33), - (-34.5, -34), - (10.999, 10), - (-5.99, -5), - (4.0e10, 40_000_000_000), - (4.0e-10, 0), - (f32::NAN, i64::MIN), - (2147483500.1, 2147483520), - (9.223371e18, 9223370937343148032), - (9.223372e18, i64::MIN), - ]; - for i in 0..inputs.len() { - let (xi, e) = inputs[i]; - let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); - let r = _mm_cvttss_si64(x); - assert_eq!( - e, r, - "TestCase #{} _mm_cvttss_si64({:?}) = {}, expected: {}", - i, x, r, e - ); - } - } - #[cfg(target_arch = "x86_64")] - test_mm_cvttss_si64(); - - // Intrinsic only available on x86_64 - #[cfg(target_arch = "x86_64")] - #[target_feature(enable = "sse")] - unsafe fn test_mm_cvtsi64_ss() { - let inputs = &[ - (4555i64, 4555.0f32), - (322223333, 322223330.0), - (-432, -432.0), - (-322223333, -322223330.0), - (9223372036854775807, 9.223372e18), - (-9223372036854775808, -9.223372e18), - ]; - - for i in 0..inputs.len() { - let (x, f) = inputs[i]; - let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); - let r = _mm_cvtsi64_ss(a, x); - let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); - assert_eq_m128(e, r); - } - } - #[cfg(target_arch = "x86_64")] - test_mm_cvtsi64_ss(); - - #[target_feature(enable = "sse")] - unsafe fn test_mm_movemask_ps() { - let r = _mm_movemask_ps(_mm_setr_ps(-1.0, 5.0, -5.0, 0.0)); - assert_eq!(r, 0b0101); - - let r = _mm_movemask_ps(_mm_setr_ps(-1.0, -5.0, -5.0, 0.0)); - assert_eq!(r, 0b0111); - } - test_mm_movemask_ps(); - - let x = 0i8; - _mm_prefetch(&x, _MM_HINT_T0); - _mm_prefetch(&x, _MM_HINT_T1); - _mm_prefetch(&x, _MM_HINT_T2); - _mm_prefetch(&x, _MM_HINT_NTA); - _mm_prefetch(&x, _MM_HINT_ET0); - _mm_prefetch(&x, _MM_HINT_ET1); - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs deleted file mode 100644 index e0088b9eb24..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs +++ /dev/null @@ -1,848 +0,0 @@ -fn main() { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - assert!(is_x86_feature_detected!("sse2")); - - unsafe { - tests::test_sse2(); - } - } -} - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod tests { - #[cfg(target_arch = "x86")] - use std::arch::x86::*; - #[cfg(target_arch = "x86_64")] - use std::arch::x86_64::*; - use std::f64::NAN; - use std::mem::transmute; - - #[target_feature(enable = "sse2")] - unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { - _mm_set_epi64x(b, a) - } - - #[target_feature(enable = "sse2")] - pub(super) unsafe fn test_sse2() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86{,_64}/sse2.rs - - unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { - _mm_set_epi64x(b, a) - } - - #[track_caller] - #[target_feature(enable = "sse")] - unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } - } - - #[track_caller] - #[target_feature(enable = "sse2")] - unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) - } - - #[track_caller] - #[target_feature(enable = "sse2")] - unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { - if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { - panic!("{:?} != {:?}", a, b); - } - } - - fn test_mm_pause() { - unsafe { _mm_pause() } - } - test_mm_pause(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_avg_epu8() { - let (a, b) = (_mm_set1_epi8(3), _mm_set1_epi8(9)); - let r = _mm_avg_epu8(a, b); - assert_eq_m128i(r, _mm_set1_epi8(6)); - } - test_mm_avg_epu8(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_avg_epu16() { - let (a, b) = (_mm_set1_epi16(3), _mm_set1_epi16(9)); - let r = _mm_avg_epu16(a, b); - assert_eq_m128i(r, _mm_set1_epi16(6)); - } - test_mm_avg_epu16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_madd_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(9, 10, 11, 12, 13, 14, 15, 16); - let r = _mm_madd_epi16(a, b); - let e = _mm_setr_epi32(29, 81, 149, 233); - assert_eq_m128i(r, e); - - let a = - _mm_setr_epi16(i16::MAX, i16::MAX, i16::MIN, i16::MIN, i16::MIN, i16::MAX, 0, 0); - let b = - _mm_setr_epi16(i16::MAX, i16::MAX, i16::MIN, i16::MIN, i16::MAX, i16::MIN, 0, 0); - let r = _mm_madd_epi16(a, b); - let e = _mm_setr_epi32(0x7FFE0002, i32::MIN, -0x7FFF0000, 0); - assert_eq_m128i(r, e); - } - test_mm_madd_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_mulhi_epi16() { - let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(-1001)); - let r = _mm_mulhi_epi16(a, b); - assert_eq_m128i(r, _mm_set1_epi16(-16)); - } - test_mm_mulhi_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_mulhi_epu16() { - let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(1001)); - let r = _mm_mulhi_epu16(a, b); - assert_eq_m128i(r, _mm_set1_epi16(15)); - } - test_mm_mulhi_epu16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_mul_epu32() { - let a = _mm_setr_epi64x(1_000_000_000, 1 << 34); - let b = _mm_setr_epi64x(1_000_000_000, 1 << 35); - let r = _mm_mul_epu32(a, b); - let e = _mm_setr_epi64x(1_000_000_000 * 1_000_000_000, 0); - assert_eq_m128i(r, e); - } - test_mm_mul_epu32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sad_epu8() { - #[rustfmt::skip] - let a = _mm_setr_epi8( - 255u8 as i8, 254u8 as i8, 253u8 as i8, 252u8 as i8, - 1, 2, 3, 4, - 155u8 as i8, 154u8 as i8, 153u8 as i8, 152u8 as i8, - 1, 2, 3, 4, - ); - let b = _mm_setr_epi8(0, 0, 0, 0, 2, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2); - let r = _mm_sad_epu8(a, b); - let e = _mm_setr_epi64x(1020, 614); - assert_eq_m128i(r, e); - } - test_mm_sad_epu8(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sll_epi16() { - let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); - let r = _mm_sll_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i( - r, - _mm_setr_epi16(0xCC0, -0xCC0, 0xDD0, -0xDD0, 0xEE0, -0xEE0, 0xFF0, -0xFF0), - ); - let r = _mm_sll_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_sll_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m128i(r, _mm_set1_epi16(0)); - let r = _mm_sll_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi16(0)); - } - test_mm_sll_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_srl_epi16() { - let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); - let r = _mm_srl_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_setr_epi16(0xC, 0xFF3, 0xD, 0xFF2, 0xE, 0xFF1, 0xF, 0xFF0)); - let r = _mm_srl_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_srl_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m128i(r, _mm_set1_epi16(0)); - let r = _mm_srl_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi16(0)); - } - test_mm_srl_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sra_epi16() { - let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); - let r = _mm_sra_epi16(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_setr_epi16(0xC, -0xD, 0xD, -0xE, 0xE, -0xF, 0xF, -0x10)); - let r = _mm_sra_epi16(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_sra_epi16(a, _mm_set_epi64x(0, 16)); - assert_eq_m128i(r, _mm_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1)); - let r = _mm_sra_epi16(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1)); - } - test_mm_sra_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sll_epi32() { - let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm_sll_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_setr_epi32(0xEEEE0, -0xEEEE0, 0xFFFF0, -0xFFFF0)); - let r = _mm_sll_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_sll_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m128i(r, _mm_set1_epi32(0)); - let r = _mm_sll_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi32(0)); - } - test_mm_sll_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_srl_epi32() { - let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm_srl_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_setr_epi32(0xEEE, 0xFFFF111, 0xFFF, 0xFFFF000)); - let r = _mm_srl_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_srl_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m128i(r, _mm_set1_epi32(0)); - let r = _mm_srl_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi32(0)); - } - test_mm_srl_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sra_epi32() { - let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); - let r = _mm_sra_epi32(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_setr_epi32(0xEEE, -0xEEF, 0xFFF, -0x1000)); - let r = _mm_sra_epi32(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_sra_epi32(a, _mm_set_epi64x(0, 32)); - assert_eq_m128i(r, _mm_setr_epi32(0, -1, 0, -1)); - let r = _mm_sra_epi32(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_setr_epi32(0, -1, 0, -1)); - } - test_mm_sra_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sll_epi64() { - let a = _mm_set_epi64x(0xFFFFFFFF, -0xFFFFFFFF); - let r = _mm_sll_epi64(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_set_epi64x(0xFFFFFFFF0, -0xFFFFFFFF0)); - let r = _mm_sll_epi64(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_sll_epi64(a, _mm_set_epi64x(0, 64)); - assert_eq_m128i(r, _mm_set1_epi64x(0)); - let r = _mm_sll_epi64(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi64x(0)); - } - test_mm_sll_epi64(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_srl_epi64() { - let a = _mm_set_epi64x(0xFFFFFFFF, -0xFFFFFFFF); - let r = _mm_srl_epi64(a, _mm_set_epi64x(0, 4)); - assert_eq_m128i(r, _mm_set_epi64x(0xFFFFFFF, 0xFFFFFFFF0000000)); - let r = _mm_srl_epi64(a, _mm_set_epi64x(4, 0)); - assert_eq_m128i(r, a); - let r = _mm_srl_epi64(a, _mm_set_epi64x(0, 64)); - assert_eq_m128i(r, _mm_set1_epi64x(0)); - let r = _mm_srl_epi64(a, _mm_set_epi64x(0, i64::MAX)); - assert_eq_m128i(r, _mm_set1_epi64x(0)); - } - test_mm_srl_epi64(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtepi32_ps() { - let a = _mm_setr_epi32(1, 2, 3, 4); - let r = _mm_cvtepi32_ps(a); - assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 3.0, 4.0)); - } - test_mm_cvtepi32_ps(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtps_epi32() { - let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_cvtps_epi32(a); - assert_eq_m128i(r, _mm_setr_epi32(1, 2, 3, 4)); - } - test_mm_cvtps_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvttps_epi32() { - let a = _mm_setr_ps(-1.1, 2.2, -3.3, 6.6); - let r = _mm_cvttps_epi32(a); - assert_eq_m128i(r, _mm_setr_epi32(-1, 2, -3, 6)); - - let a = _mm_setr_ps(f32::NEG_INFINITY, f32::INFINITY, f32::MIN, f32::MAX); - let r = _mm_cvttps_epi32(a); - assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, i32::MIN, i32::MIN)); - } - test_mm_cvttps_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_packs_epi16() { - let a = _mm_setr_epi16(0x80, -0x81, 0, 0, 0, 0, 0, 0); - let b = _mm_setr_epi16(0, 0, 0, 0, 0, 0, -0x81, 0x80); - let r = _mm_packs_epi16(a, b); - assert_eq_m128i( - r, - _mm_setr_epi8(0x7F, -0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0x80, 0x7F), - ); - } - test_mm_packs_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_packus_epi16() { - let a = _mm_setr_epi16(0x100, -1, 0, 0, 0, 0, 0, 0); - let b = _mm_setr_epi16(0, 0, 0, 0, 0, 0, -1, 0x100); - let r = _mm_packus_epi16(a, b); - assert_eq_m128i(r, _mm_setr_epi8(!0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, !0)); - } - test_mm_packus_epi16(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_packs_epi32() { - let a = _mm_setr_epi32(0x8000, -0x8001, 0, 0); - let b = _mm_setr_epi32(0, 0, -0x8001, 0x8000); - let r = _mm_packs_epi32(a, b); - assert_eq_m128i(r, _mm_setr_epi16(0x7FFF, -0x8000, 0, 0, 0, 0, -0x8000, 0x7FFF)); - } - test_mm_packs_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_min_sd() { - let a = _mm_setr_pd(1.0, 2.0); - let b = _mm_setr_pd(5.0, 10.0); - let r = _mm_min_sd(a, b); - assert_eq_m128d(r, _mm_setr_pd(1.0, 2.0)); - } - test_mm_min_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_min_pd() { - let a = _mm_setr_pd(-1.0, 5.0); - let b = _mm_setr_pd(-100.0, 20.0); - let r = _mm_min_pd(a, b); - assert_eq_m128d(r, _mm_setr_pd(-100.0, 5.0)); - - // `_mm_min_pd` can **not** be implemented using the `simd_min` rust intrinsic because - // the semantics of `simd_min` are different to those of `_mm_min_pd` regarding handling - // of `-0.0`. - let a = _mm_setr_pd(-0.0, 0.0); - let b = _mm_setr_pd(0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_min_pd(a, b)); - let r2: [u8; 16] = transmute(_mm_min_pd(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); - assert_eq!(r1, b); - assert_eq!(r2, a); - assert_ne!(a, b); // sanity check that -0.0 is actually present - } - test_mm_min_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_max_sd() { - let a = _mm_setr_pd(1.0, 2.0); - let b = _mm_setr_pd(5.0, 10.0); - let r = _mm_max_sd(a, b); - assert_eq_m128d(r, _mm_setr_pd(5.0, 2.0)); - } - test_mm_max_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_max_pd() { - let a = _mm_setr_pd(-1.0, 5.0); - let b = _mm_setr_pd(-100.0, 20.0); - let r = _mm_max_pd(a, b); - assert_eq_m128d(r, _mm_setr_pd(-1.0, 20.0)); - - // `_mm_max_pd` can **not** be implemented using the `simd_max` rust intrinsic because - // the semantics of `simd_max` are different to those of `_mm_max_pd` regarding handling - // of `-0.0`. - let a = _mm_setr_pd(-0.0, 0.0); - let b = _mm_setr_pd(0.0, 0.0); - let r1: [u8; 16] = transmute(_mm_max_pd(a, b)); - let r2: [u8; 16] = transmute(_mm_max_pd(b, a)); - let a: [u8; 16] = transmute(a); - let b: [u8; 16] = transmute(b); - assert_eq!(r1, b); - assert_eq!(r2, a); - assert_ne!(a, b); // sanity check that -0.0 is actually present - } - test_mm_max_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sqrt_sd() { - let a = _mm_setr_pd(1.0, 2.0); - let b = _mm_setr_pd(5.0, 10.0); - let r = _mm_sqrt_sd(a, b); - assert_eq_m128d(r, _mm_setr_pd(5.0f64.sqrt(), 2.0)); - } - test_mm_sqrt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_sqrt_pd() { - let r = _mm_sqrt_pd(_mm_setr_pd(1.0, 2.0)); - assert_eq_m128d(r, _mm_setr_pd(1.0f64.sqrt(), 2.0f64.sqrt())); - } - test_mm_sqrt_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpeq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpeq_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpeq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmplt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmplt_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmplt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmple_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmple_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmple_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpgt_sd() { - let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpgt_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpgt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpge_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpge_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpge_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpord_sd() { - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpord_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpord_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpunord_sd() { - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpunord_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpunord_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpneq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(!0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpneq_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpneq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnlt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpnlt_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnlt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnle_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpnle_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnle_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpngt_sd() { - let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpngt_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpngt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnge_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, transmute(2.0f64)); - let r = transmute::<_, __m128i>(_mm_cmpnge_sd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnge_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpeq_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, 0); - let r = transmute::<_, __m128i>(_mm_cmpeq_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpeq_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmplt_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, !0); - let r = transmute::<_, __m128i>(_mm_cmplt_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmplt_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmple_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, !0); - let r = transmute::<_, __m128i>(_mm_cmple_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmple_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpgt_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, 0); - let r = transmute::<_, __m128i>(_mm_cmpgt_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpgt_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpge_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(!0, 0); - let r = transmute::<_, __m128i>(_mm_cmpge_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpge_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpord_pd() { - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(0, !0); - let r = transmute::<_, __m128i>(_mm_cmpord_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpord_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpunord_pd() { - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(!0, 0); - let r = transmute::<_, __m128i>(_mm_cmpunord_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpunord_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpneq_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(!0, !0); - let r = transmute::<_, __m128i>(_mm_cmpneq_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpneq_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnlt_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); - let e = _mm_setr_epi64x(0, 0); - let r = transmute::<_, __m128i>(_mm_cmpnlt_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnlt_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnle_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, 0); - let r = transmute::<_, __m128i>(_mm_cmpnle_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnle_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpngt_pd() { - let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, !0); - let r = transmute::<_, __m128i>(_mm_cmpngt_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpngt_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cmpnge_pd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - let e = _mm_setr_epi64x(0, !0); - let r = transmute::<_, __m128i>(_mm_cmpnge_pd(a, b)); - assert_eq_m128i(r, e); - } - test_mm_cmpnge_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comieq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comieq_sd(a, b) != 0); - - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comieq_sd(a, b) == 0); - } - test_mm_comieq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comilt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comilt_sd(a, b) == 0); - } - test_mm_comilt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comile_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comile_sd(a, b) != 0); - } - test_mm_comile_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comigt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comigt_sd(a, b) == 0); - } - test_mm_comigt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comige_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comige_sd(a, b) != 0); - } - test_mm_comige_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_comineq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_comineq_sd(a, b) == 0); - } - test_mm_comineq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomieq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomieq_sd(a, b) != 0); - - let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(NAN, 3.0)); - assert!(_mm_ucomieq_sd(a, b) == 0); - } - test_mm_ucomieq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomilt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomilt_sd(a, b) == 0); - } - test_mm_ucomilt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomile_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomile_sd(a, b) != 0); - } - test_mm_ucomile_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomigt_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomigt_sd(a, b) == 0); - } - test_mm_ucomigt_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomige_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomige_sd(a, b) != 0); - } - test_mm_ucomige_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_ucomineq_sd() { - let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); - assert!(_mm_ucomineq_sd(a, b) == 0); - } - test_mm_ucomineq_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtpd_ps() { - let r = _mm_cvtpd_ps(_mm_setr_pd(-1.0, 5.0)); - assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, 0.0)); - - let r = _mm_cvtpd_ps(_mm_setr_pd(-1.0, -5.0)); - assert_eq_m128(r, _mm_setr_ps(-1.0, -5.0, 0.0, 0.0)); - - let r = _mm_cvtpd_ps(_mm_setr_pd(f64::MAX, f64::MIN)); - assert_eq_m128(r, _mm_setr_ps(f32::INFINITY, f32::NEG_INFINITY, 0.0, 0.0)); - - let r = _mm_cvtpd_ps(_mm_setr_pd(f32::MAX as f64, f32::MIN as f64)); - assert_eq_m128(r, _mm_setr_ps(f32::MAX, f32::MIN, 0.0, 0.0)); - } - test_mm_cvtpd_ps(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtps_pd() { - let r = _mm_cvtps_pd(_mm_setr_ps(-1.0, 2.0, -3.0, 5.0)); - assert_eq_m128d(r, _mm_setr_pd(-1.0, 2.0)); - - let r = _mm_cvtps_pd(_mm_setr_ps(f32::MAX, f32::INFINITY, f32::NEG_INFINITY, f32::MIN)); - assert_eq_m128d(r, _mm_setr_pd(f32::MAX as f64, f64::INFINITY)); - } - test_mm_cvtps_pd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtpd_epi32() { - let r = _mm_cvtpd_epi32(_mm_setr_pd(-1.0, 5.0)); - assert_eq_m128i(r, _mm_setr_epi32(-1, 5, 0, 0)); - - let r = _mm_cvtpd_epi32(_mm_setr_pd(-1.0, -5.0)); - assert_eq_m128i(r, _mm_setr_epi32(-1, -5, 0, 0)); - - let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::MAX, f64::MIN)); - assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); - - let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::INFINITY, f64::NEG_INFINITY)); - assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); - - let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::NAN, f64::NAN)); - assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); - } - test_mm_cvtpd_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvttpd_epi32() { - let a = _mm_setr_pd(-1.1, 2.2); - let r = _mm_cvttpd_epi32(a); - assert_eq_m128i(r, _mm_setr_epi32(-1, 2, 0, 0)); - - let a = _mm_setr_pd(f64::NEG_INFINITY, f64::NAN); - let r = _mm_cvttpd_epi32(a); - assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); - } - test_mm_cvttpd_epi32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtsd_si32() { - let r = _mm_cvtsd_si32(_mm_setr_pd(-2.0, 5.0)); - assert_eq!(r, -2); - - let r = _mm_cvtsd_si32(_mm_setr_pd(f64::MAX, f64::MIN)); - assert_eq!(r, i32::MIN); - - let r = _mm_cvtsd_si32(_mm_setr_pd(f64::NAN, f64::NAN)); - assert_eq!(r, i32::MIN); - } - test_mm_cvtsd_si32(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvttsd_si32() { - let a = _mm_setr_pd(-1.1, 2.2); - let r = _mm_cvttsd_si32(a); - assert_eq!(r, -1); - - let a = _mm_setr_pd(f64::NEG_INFINITY, f64::NAN); - let r = _mm_cvttsd_si32(a); - assert_eq!(r, i32::MIN); - } - test_mm_cvttsd_si32(); - - // Intrinsic only available on x86_64 - #[cfg(target_arch = "x86_64")] - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtsd_si64() { - let r = _mm_cvtsd_si64(_mm_setr_pd(-2.0, 5.0)); - assert_eq!(r, -2_i64); - - let r = _mm_cvtsd_si64(_mm_setr_pd(f64::MAX, f64::MIN)); - assert_eq!(r, i64::MIN); - } - #[cfg(target_arch = "x86_64")] - test_mm_cvtsd_si64(); - - // Intrinsic only available on x86_64 - #[cfg(target_arch = "x86_64")] - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvttsd_si64() { - let a = _mm_setr_pd(-1.1, 2.2); - let r = _mm_cvttsd_si64(a); - assert_eq!(r, -1_i64); - } - #[cfg(target_arch = "x86_64")] - test_mm_cvttsd_si64(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtsd_ss() { - let a = _mm_setr_ps(-1.1, -2.2, 3.3, 4.4); - let b = _mm_setr_pd(2.0, -5.0); - - let r = _mm_cvtsd_ss(a, b); - - assert_eq_m128(r, _mm_setr_ps(2.0, -2.2, 3.3, 4.4)); - - let a = _mm_setr_ps(-1.1, f32::NEG_INFINITY, f32::MAX, f32::NEG_INFINITY); - let b = _mm_setr_pd(f64::INFINITY, -5.0); - - let r = _mm_cvtsd_ss(a, b); - - assert_eq_m128( - r, - _mm_setr_ps(f32::INFINITY, f32::NEG_INFINITY, f32::MAX, f32::NEG_INFINITY), - ); - } - test_mm_cvtsd_ss(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_cvtss_sd() { - let a = _mm_setr_pd(-1.1, 2.2); - let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - - let r = _mm_cvtss_sd(a, b); - assert_eq_m128d(r, _mm_setr_pd(1.0, 2.2)); - - let a = _mm_setr_pd(-1.1, f64::INFINITY); - let b = _mm_setr_ps(f32::NEG_INFINITY, 2.0, 3.0, 4.0); - - let r = _mm_cvtss_sd(a, b); - assert_eq_m128d(r, _mm_setr_pd(f64::NEG_INFINITY, f64::INFINITY)); - } - test_mm_cvtss_sd(); - - #[target_feature(enable = "sse2")] - unsafe fn test_mm_movemask_pd() { - let r = _mm_movemask_pd(_mm_setr_pd(-1.0, 5.0)); - assert_eq!(r, 0b01); - - let r = _mm_movemask_pd(_mm_setr_pd(-1.0, -5.0)); - assert_eq!(r, 0b11); - } - test_mm_movemask_pd(); - } -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs deleted file mode 100644 index 7566be4431b..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-sse3-ssse3.rs +++ /dev/null @@ -1,395 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -// SSSE3 implicitly enables SSE3 -//@compile-flags: -C target-feature=+ssse3 - -use core::mem::transmute; -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; - -fn main() { - // SSSE3 implicitly enables SSE3, still check it to be sure - assert!(is_x86_feature_detected!("sse3")); - assert!(is_x86_feature_detected!("ssse3")); - - unsafe { - test_sse3(); - test_ssse3(); - } -} - -#[target_feature(enable = "sse3")] -unsafe fn test_sse3() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/sse3.rs - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_addsub_ps() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_addsub_ps(a, b); - assert_eq_m128(r, _mm_setr_ps(99.0, 25.0, 0.0, -15.0)); - } - test_mm_addsub_ps(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_addsub_pd() { - let a = _mm_setr_pd(-1.0, 5.0); - let b = _mm_setr_pd(-100.0, 20.0); - let r = _mm_addsub_pd(a, b); - assert_eq_m128d(r, _mm_setr_pd(99.0, 25.0)); - } - test_mm_addsub_pd(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_hadd_ps() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_hadd_ps(a, b); - assert_eq_m128(r, _mm_setr_ps(4.0, -10.0, -80.0, -5.0)); - } - test_mm_hadd_ps(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_hadd_pd() { - let a = _mm_setr_pd(-1.0, 5.0); - let b = _mm_setr_pd(-100.0, 20.0); - let r = _mm_hadd_pd(a, b); - assert_eq_m128d(r, _mm_setr_pd(4.0, -80.0)); - } - test_mm_hadd_pd(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_hsub_ps() { - let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); - let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); - let r = _mm_hsub_ps(a, b); - assert_eq_m128(r, _mm_setr_ps(-6.0, 10.0, -120.0, 5.0)); - } - test_mm_hsub_ps(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_hsub_pd() { - let a = _mm_setr_pd(-1.0, 5.0); - let b = _mm_setr_pd(-100.0, 20.0); - let r = _mm_hsub_pd(a, b); - assert_eq_m128d(r, _mm_setr_pd(-6.0, -120.0)); - } - test_mm_hsub_pd(); - - #[target_feature(enable = "sse3")] - unsafe fn test_mm_lddqu_si128() { - let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let r = _mm_lddqu_si128(&a); - assert_eq_m128i(a, r); - } - test_mm_lddqu_si128(); -} - -#[target_feature(enable = "ssse3")] -unsafe fn test_ssse3() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/ssse3.rs - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_abs_epi8() { - let r = _mm_abs_epi8(_mm_set1_epi8(-5)); - assert_eq_m128i(r, _mm_set1_epi8(5)); - } - test_mm_abs_epi8(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_abs_epi16() { - let r = _mm_abs_epi16(_mm_set1_epi16(-5)); - assert_eq_m128i(r, _mm_set1_epi16(5)); - } - test_mm_abs_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_abs_epi32() { - let r = _mm_abs_epi32(_mm_set1_epi32(-5)); - assert_eq_m128i(r, _mm_set1_epi32(5)); - } - test_mm_abs_epi32(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_shuffle_epi8() { - let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b = _mm_setr_epi8(4, 128_u8 as i8, 4, 3, 24, 12, 6, 19, 12, 5, 5, 10, 4, 1, 8, 0); - let expected = _mm_setr_epi8(5, 0, 5, 4, 9, 13, 7, 4, 13, 6, 6, 11, 5, 2, 9, 1); - let r = _mm_shuffle_epi8(a, b); - assert_eq_m128i(r, expected); - - // Test indices greater than 15 wrapping around - let b = _mm_add_epi8(b, _mm_set1_epi8(32)); - let r = _mm_shuffle_epi8(a, b); - assert_eq_m128i(r, expected); - } - test_mm_shuffle_epi8(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hadd_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(4, 128, 4, 3, 24, 12, 6, 19); - let expected = _mm_setr_epi16(3, 7, 11, 15, 132, 7, 36, 25); - let r = _mm_hadd_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test wrapping on overflow - let a = _mm_setr_epi16(i16::MAX, 1, i16::MAX, 2, i16::MAX, 3, i16::MAX, 4); - let b = _mm_setr_epi16(i16::MIN, -1, i16::MIN, -2, i16::MIN, -3, i16::MIN, -4); - let expected = _mm_setr_epi16( - i16::MIN, - i16::MIN + 1, - i16::MIN + 2, - i16::MIN + 3, - i16::MAX, - i16::MAX - 1, - i16::MAX - 2, - i16::MAX - 3, - ); - let r = _mm_hadd_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hadd_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hadds_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(4, 128, 4, 3, 32767, 1, -32768, -1); - let expected = _mm_setr_epi16(3, 7, 11, 15, 132, 7, 32767, -32768); - let r = _mm_hadds_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test saturating on overflow - let a = _mm_setr_epi16(i16::MAX, 1, i16::MAX, 2, i16::MAX, 3, i16::MAX, 4); - let b = _mm_setr_epi16(i16::MIN, -1, i16::MIN, -2, i16::MIN, -3, i16::MIN, -4); - let expected = _mm_setr_epi16( - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - ); - let r = _mm_hadds_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hadds_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hadd_epi32() { - let a = _mm_setr_epi32(1, 2, 3, 4); - let b = _mm_setr_epi32(4, 128, 4, 3); - let expected = _mm_setr_epi32(3, 7, 132, 7); - let r = _mm_hadd_epi32(a, b); - assert_eq_m128i(r, expected); - - // Test wrapping on overflow - let a = _mm_setr_epi32(i32::MAX, 1, i32::MAX, 2); - let b = _mm_setr_epi32(i32::MIN, -1, i32::MIN, -2); - let expected = _mm_setr_epi32(i32::MIN, i32::MIN + 1, i32::MAX, i32::MAX - 1); - let r = _mm_hadd_epi32(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hadd_epi32(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hsub_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(4, 128, 4, 3, 24, 12, 6, 19); - let expected = _mm_setr_epi16(-1, -1, -1, -1, -124, 1, 12, -13); - let r = _mm_hsub_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test wrapping on overflow - let a = _mm_setr_epi16(i16::MAX, -1, i16::MAX, -2, i16::MAX, -3, i16::MAX, -4); - let b = _mm_setr_epi16(i16::MIN, 1, i16::MIN, 2, i16::MIN, 3, i16::MIN, 4); - let expected = _mm_setr_epi16( - i16::MIN, - i16::MIN + 1, - i16::MIN + 2, - i16::MIN + 3, - i16::MAX, - i16::MAX - 1, - i16::MAX - 2, - i16::MAX - 3, - ); - let r = _mm_hsub_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hsub_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hsubs_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(4, 128, 4, 3, 32767, -1, -32768, 1); - let expected = _mm_setr_epi16(-1, -1, -1, -1, -124, 1, 32767, -32768); - let r = _mm_hsubs_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test saturating on overflow - let a = _mm_setr_epi16(i16::MAX, -1, i16::MAX, -2, i16::MAX, -3, i16::MAX, -4); - let b = _mm_setr_epi16(i16::MIN, 1, i16::MIN, 2, i16::MIN, 3, i16::MIN, 4); - let expected = _mm_setr_epi16( - i16::MAX, - i16::MAX, - i16::MAX, - i16::MAX, - i16::MIN, - i16::MIN, - i16::MIN, - i16::MIN, - ); - let r = _mm_hsubs_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hsubs_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_hsub_epi32() { - let a = _mm_setr_epi32(1, 2, 3, 4); - let b = _mm_setr_epi32(4, 128, 4, 3); - let expected = _mm_setr_epi32(-1, -1, -124, 1); - let r = _mm_hsub_epi32(a, b); - assert_eq_m128i(r, expected); - - // Test wrapping on overflow - let a = _mm_setr_epi32(i32::MAX, -1, i32::MAX, -2); - let b = _mm_setr_epi32(i32::MIN, 1, i32::MIN, 2); - let expected = _mm_setr_epi32(i32::MIN, i32::MIN + 1, i32::MAX, i32::MAX - 1); - let r = _mm_hsub_epi32(a, b); - assert_eq_m128i(r, expected); - } - test_mm_hsub_epi32(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_maddubs_epi16() { - let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - let b = _mm_setr_epi8(4, 63, 4, 3, 24, 12, 6, 19, 12, 5, 5, 10, 4, 1, 8, 0); - let expected = _mm_setr_epi16(130, 24, 192, 194, 158, 175, 66, 120); - let r = _mm_maddubs_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test widening and saturation - let a = _mm_setr_epi8( - u8::MAX as i8, - u8::MAX as i8, - u8::MAX as i8, - u8::MAX as i8, - u8::MAX as i8, - u8::MAX as i8, - 100, - 100, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ); - let b = _mm_setr_epi8( - i8::MAX, - i8::MAX, - i8::MAX, - i8::MIN, - i8::MIN, - i8::MIN, - 50, - 15, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ); - let expected = _mm_setr_epi16(i16::MAX, -255, i16::MIN, 6500, 0, 0, 0, 0); - let r = _mm_maddubs_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_maddubs_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_mulhrs_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); - let b = _mm_setr_epi16(4, 128, 4, 3, 32767, -1, -32768, 1); - let expected = _mm_setr_epi16(0, 0, 0, 0, 5, 0, -7, 0); - let r = _mm_mulhrs_epi16(a, b); - assert_eq_m128i(r, expected); - - // Test extreme values - let a = _mm_setr_epi16(i16::MAX, i16::MIN, i16::MIN, 0, 0, 0, 0, 0); - let b = _mm_setr_epi16(i16::MAX, i16::MIN, i16::MAX, 0, 0, 0, 0, 0); - let expected = _mm_setr_epi16(i16::MAX - 1, i16::MIN, -i16::MAX, 0, 0, 0, 0, 0); - let r = _mm_mulhrs_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_mulhrs_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_sign_epi8() { - let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -14, -15, 16); - let b = _mm_setr_epi8(4, 63, -4, 3, 24, 12, -6, -19, 12, 5, -5, 10, 4, 1, -8, 0); - let expected = _mm_setr_epi8(1, 2, -3, 4, 5, 6, -7, -8, 9, 10, -11, 12, 13, -14, 15, 0); - let r = _mm_sign_epi8(a, b); - assert_eq_m128i(r, expected); - } - test_mm_sign_epi8(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_sign_epi16() { - let a = _mm_setr_epi16(1, 2, 3, 4, -5, -6, 7, 8); - let b = _mm_setr_epi16(4, 128, 0, 3, 1, -1, -2, 1); - let expected = _mm_setr_epi16(1, 2, 0, 4, -5, 6, -7, 8); - let r = _mm_sign_epi16(a, b); - assert_eq_m128i(r, expected); - } - test_mm_sign_epi16(); - - #[target_feature(enable = "ssse3")] - unsafe fn test_mm_sign_epi32() { - let a = _mm_setr_epi32(-1, 2, 3, 4); - let b = _mm_setr_epi32(1, -1, 1, 0); - let expected = _mm_setr_epi32(-1, -2, 3, 0); - let r = _mm_sign_epi32(a, b); - assert_eq_m128i(r, expected); - } - test_mm_sign_epi32(); -} - -#[track_caller] -#[target_feature(enable = "sse")] -unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { - if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -pub unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse41.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse41.rs deleted file mode 100644 index 06607f3fd59..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86-sse41.rs +++ /dev/null @@ -1,548 +0,0 @@ -// Ignore everything except x86 and x86_64 -// Any new targets that are added to CI should be ignored here. -// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) -//@ignore-target-aarch64 -//@ignore-target-arm -//@ignore-target-avr -//@ignore-target-s390x -//@ignore-target-thumbv7em -//@ignore-target-wasm32 -//@compile-flags: -C target-feature=+sse4.1 - -#[cfg(target_arch = "x86")] -use std::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use std::arch::x86_64::*; -use std::mem::transmute; - -fn main() { - assert!(is_x86_feature_detected!("sse4.1")); - - unsafe { - test_sse41(); - } -} - -#[target_feature(enable = "sse4.1")] -unsafe fn test_sse41() { - // Mostly copied from library/stdarch/crates/core_arch/src/x86/sse41.rs - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_insert_ps() { - let a = _mm_set1_ps(1.0); - let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_insert_ps::<0b11_00_1100>(a, b); - let e = _mm_setr_ps(4.0, 1.0, 0.0, 0.0); - assert_eq_m128(r, e); - - // Zeroing takes precedence over copied value - let a = _mm_set1_ps(1.0); - let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); - let r = _mm_insert_ps::<0b11_00_0001>(a, b); - let e = _mm_setr_ps(0.0, 1.0, 1.0, 1.0); - assert_eq_m128(r, e); - } - test_mm_insert_ps(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_packus_epi32() { - let a = _mm_setr_epi32(1, 2, 3, 4); - let b = _mm_setr_epi32(-1, -2, -3, -4); - let r = _mm_packus_epi32(a, b); - let e = _mm_setr_epi16(1, 2, 3, 4, 0, 0, 0, 0); - assert_eq_m128i(r, e); - } - test_mm_packus_epi32(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_dp_pd() { - let a = _mm_setr_pd(2.0, 3.0); - let b = _mm_setr_pd(1.0, 4.0); - let e = _mm_setr_pd(14.0, 0.0); - assert_eq_m128d(_mm_dp_pd::<0b00110001>(a, b), e); - } - test_mm_dp_pd(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_dp_ps() { - let a = _mm_setr_ps(2.0, 3.0, 1.0, 10.0); - let b = _mm_setr_ps(1.0, 4.0, 0.5, 10.0); - let e = _mm_setr_ps(14.5, 0.0, 14.5, 0.0); - assert_eq_m128(_mm_dp_ps::<0b01110101>(a, b), e); - } - test_mm_dp_ps(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_nearest_f32() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f32, res: f32) { - let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); - let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); - let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); - let r = _mm_round_ss::<_MM_FROUND_TO_NEAREST_INT>(a, b); - assert_eq_m128(r, e); - // Assume round-to-nearest by default - let r = _mm_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); - assert_eq_m128(r, e); - - let a = _mm_set1_ps(x); - let e = _mm_set1_ps(res); - let r = _mm_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m128(r, e); - // Assume round-to-nearest by default - let r = _mm_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m128(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); - let e = _mm_setr_ps(2.0, 4.0, 6.0, 8.0); - let r = _mm_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m128(r, e); - // Assume round-to-nearest by default - let r = _mm_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m128(r, e); - } - test_round_nearest_f32(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_floor_f32() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f32, res: f32) { - let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); - let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); - let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); - let r = _mm_floor_ss(a, b); - assert_eq_m128(r, e); - let r = _mm_round_ss::<_MM_FROUND_TO_NEG_INF>(a, b); - assert_eq_m128(r, e); - - let a = _mm_set1_ps(x); - let e = _mm_set1_ps(res); - let r = _mm_floor_ps(a); - assert_eq_m128(r, e); - let r = _mm_round_ps::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m128(r, e); - } - - // Test rounding direction - test(-2.5, -3.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -2.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); - let e = _mm_setr_ps(1.0, 3.0, 5.0, 7.0); - let r = _mm_floor_ps(a); - assert_eq_m128(r, e); - let r = _mm_round_ps::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m128(r, e); - } - test_round_floor_f32(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_ceil_f32() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f32, res: f32) { - let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); - let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); - let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); - let r = _mm_ceil_ss(a, b); - assert_eq_m128(r, e); - let r = _mm_round_ss::<_MM_FROUND_TO_POS_INF>(a, b); - assert_eq_m128(r, e); - - let a = _mm_set1_ps(x); - let e = _mm_set1_ps(res); - let r = _mm_ceil_ps(a); - assert_eq_m128(r, e); - let r = _mm_round_ps::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m128(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 2.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 3.0); - - // Test that each element is rounded - let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); - let e = _mm_setr_ps(2.0, 4.0, 6.0, 8.0); - let r = _mm_ceil_ps(a); - assert_eq_m128(r, e); - let r = _mm_round_ps::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m128(r, e); - } - test_round_ceil_f32(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_trunc_f32() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f32, res: f32) { - let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); - let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); - let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); - let r = _mm_round_ss::<_MM_FROUND_TO_ZERO>(a, b); - assert_eq_m128(r, e); - - let a = _mm_set1_ps(x); - let e = _mm_set1_ps(res); - let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m128(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); - let e = _mm_setr_ps(1.0, 3.0, 5.0, 7.0); - let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m128(r, e); - } - test_round_trunc_f32(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_nearest_f64() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f64, res: f64) { - let a = _mm_setr_pd(3.5, 2.5); - let b = _mm_setr_pd(x, -1.5); - let e = _mm_setr_pd(res, 2.5); - let r = _mm_round_sd::<_MM_FROUND_TO_NEAREST_INT>(a, b); - assert_eq_m128d(r, e); - // Assume round-to-nearest by default - let r = _mm_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); - assert_eq_m128d(r, e); - - let a = _mm_set1_pd(x); - let e = _mm_set1_pd(res); - let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m128d(r, e); - // Assume round-to-nearest by default - let r = _mm_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m128d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_pd(1.5, 3.5); - let e = _mm_setr_pd(2.0, 4.0); - let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); - assert_eq_m128d(r, e); - // Assume round-to-nearest by default - let r = _mm_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); - assert_eq_m128d(r, e); - } - test_round_nearest_f64(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_floor_f64() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f64, res: f64) { - let a = _mm_setr_pd(3.5, 2.5); - let b = _mm_setr_pd(x, -1.5); - let e = _mm_setr_pd(res, 2.5); - let r = _mm_floor_sd(a, b); - assert_eq_m128d(r, e); - let r = _mm_round_sd::<_MM_FROUND_TO_NEG_INF>(a, b); - assert_eq_m128d(r, e); - - let a = _mm_set1_pd(x); - let e = _mm_set1_pd(res); - let r = _mm_floor_pd(a); - assert_eq_m128d(r, e); - let r = _mm_round_pd::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m128d(r, e); - } - - // Test rounding direction - test(-2.5, -3.0); - test(-1.75, -2.0); - test(-1.5, -2.0); - test(-1.25, -2.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_pd(1.5, 3.5); - let e = _mm_setr_pd(1.0, 3.0); - let r = _mm_floor_pd(a); - assert_eq_m128d(r, e); - let r = _mm_round_pd::<_MM_FROUND_TO_NEG_INF>(a); - assert_eq_m128d(r, e); - } - test_round_floor_f64(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_ceil_f64() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f64, res: f64) { - let a = _mm_setr_pd(3.5, 2.5); - let b = _mm_setr_pd(x, -1.5); - let e = _mm_setr_pd(res, 2.5); - let r = _mm_ceil_sd(a, b); - assert_eq_m128d(r, e); - let r = _mm_round_sd::<_MM_FROUND_TO_POS_INF>(a, b); - assert_eq_m128d(r, e); - - let a = _mm_set1_pd(x); - let e = _mm_set1_pd(res); - let r = _mm_ceil_pd(a); - assert_eq_m128d(r, e); - let r = _mm_round_pd::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m128d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 2.0); - test(1.5, 2.0); - test(1.75, 2.0); - test(2.5, 3.0); - - // Test that each element is rounded - let a = _mm_setr_pd(1.5, 3.5); - let e = _mm_setr_pd(2.0, 4.0); - let r = _mm_ceil_pd(a); - assert_eq_m128d(r, e); - let r = _mm_round_pd::<_MM_FROUND_TO_POS_INF>(a); - assert_eq_m128d(r, e); - } - test_round_ceil_f64(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_round_trunc_f64() { - #[target_feature(enable = "sse4.1")] - unsafe fn test(x: f64, res: f64) { - let a = _mm_setr_pd(3.5, 2.5); - let b = _mm_setr_pd(x, -1.5); - let e = _mm_setr_pd(res, 2.5); - let r = _mm_round_sd::<_MM_FROUND_TO_ZERO>(a, b); - assert_eq_m128d(r, e); - - let a = _mm_set1_pd(x); - let e = _mm_set1_pd(res); - let r = _mm_round_pd::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m128d(r, e); - } - - // Test rounding direction - test(-2.5, -2.0); - test(-1.75, -1.0); - test(-1.5, -1.0); - test(-1.25, -1.0); - test(-1.0, -1.0); - test(0.0, 0.0); - test(1.0, 1.0); - test(1.25, 1.0); - test(1.5, 1.0); - test(1.75, 1.0); - test(2.5, 2.0); - - // Test that each element is rounded - let a = _mm_setr_pd(1.5, 3.5); - let e = _mm_setr_pd(1.0, 3.0); - let r = _mm_round_pd::<_MM_FROUND_TO_ZERO>(a); - assert_eq_m128d(r, e); - } - test_round_trunc_f64(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_minpos_epu16() { - let a = _mm_setr_epi16(23, 18, 44, 97, 50, 13, 67, 66); - let r = _mm_minpos_epu16(a); - let e = _mm_setr_epi16(13, 5, 0, 0, 0, 0, 0, 0); - assert_eq_m128i(r, e); - - let a = _mm_setr_epi16(0, 18, 44, 97, 50, 13, 67, 66); - let r = _mm_minpos_epu16(a); - let e = _mm_setr_epi16(0, 0, 0, 0, 0, 0, 0, 0); - assert_eq_m128i(r, e); - - // Case where the minimum value is repeated - let a = _mm_setr_epi16(23, 18, 44, 97, 50, 13, 67, 13); - let r = _mm_minpos_epu16(a); - let e = _mm_setr_epi16(13, 5, 0, 0, 0, 0, 0, 0); - assert_eq_m128i(r, e); - } - test_mm_minpos_epu16(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_mpsadbw_epu8() { - let a = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - - let r = _mm_mpsadbw_epu8::<0b000>(a, a); - let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); - assert_eq_m128i(r, e); - - let r = _mm_mpsadbw_epu8::<0b001>(a, a); - let e = _mm_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12); - assert_eq_m128i(r, e); - - let r = _mm_mpsadbw_epu8::<0b100>(a, a); - let e = _mm_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44); - assert_eq_m128i(r, e); - - let r = _mm_mpsadbw_epu8::<0b101>(a, a); - let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); - assert_eq_m128i(r, e); - - let r = _mm_mpsadbw_epu8::<0b111>(a, a); - let e = _mm_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4); - assert_eq_m128i(r, e); - } - test_mm_mpsadbw_epu8(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_testz_si128() { - let a = _mm_set1_epi8(1); - let mask = _mm_set1_epi8(0); - let r = _mm_testz_si128(a, mask); - assert_eq!(r, 1); - - let a = _mm_set1_epi8(0b101); - let mask = _mm_set1_epi8(0b110); - let r = _mm_testz_si128(a, mask); - assert_eq!(r, 0); - - let a = _mm_set1_epi8(0b011); - let mask = _mm_set1_epi8(0b100); - let r = _mm_testz_si128(a, mask); - assert_eq!(r, 1); - } - test_mm_testz_si128(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_testc_si128() { - let a = _mm_set1_epi8(-1); - let mask = _mm_set1_epi8(0); - let r = _mm_testc_si128(a, mask); - assert_eq!(r, 1); - - let a = _mm_set1_epi8(0b101); - let mask = _mm_set1_epi8(0b110); - let r = _mm_testc_si128(a, mask); - assert_eq!(r, 0); - - let a = _mm_set1_epi8(0b101); - let mask = _mm_set1_epi8(0b100); - let r = _mm_testc_si128(a, mask); - assert_eq!(r, 1); - } - test_mm_testc_si128(); - - #[target_feature(enable = "sse4.1")] - unsafe fn test_mm_testnzc_si128() { - let a = _mm_set1_epi8(0); - let mask = _mm_set1_epi8(1); - let r = _mm_testnzc_si128(a, mask); - assert_eq!(r, 0); - - let a = _mm_set1_epi8(-1); - let mask = _mm_set1_epi8(0); - let r = _mm_testnzc_si128(a, mask); - assert_eq!(r, 0); - - let a = _mm_set1_epi8(0b101); - let mask = _mm_set1_epi8(0b110); - let r = _mm_testnzc_si128(a, mask); - assert_eq!(r, 1); - - let a = _mm_set1_epi8(0b101); - let mask = _mm_set1_epi8(0b101); - let r = _mm_testnzc_si128(a, mask); - assert_eq!(r, 0); - - let a = _mm_setr_epi32(0b100, 0, 0, 0b010); - let mask = _mm_setr_epi32(0b100, 0, 0, 0b110); - let r = _mm_testnzc_si128(a, mask); - assert_eq!(r, 1); - } - test_mm_testnzc_si128(); -} - -#[track_caller] -#[target_feature(enable = "sse")] -unsafe fn assert_eq_m128(a: __m128, b: __m128) { - let r = _mm_cmpeq_ps(a, b); - if _mm_movemask_ps(r) != 0b1111 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { - if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { - panic!("{:?} != {:?}", a, b); - } -} - -#[track_caller] -#[target_feature(enable = "sse2")] -pub unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { - assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) -} diff --git a/src/tools/miri/tests/pass/intrinsics-x86.rs b/src/tools/miri/tests/pass/intrinsics-x86.rs deleted file mode 100644 index 90bcdba4353..00000000000 --- a/src/tools/miri/tests/pass/intrinsics-x86.rs +++ /dev/null @@ -1,99 +0,0 @@ -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod x86 { - #[cfg(target_arch = "x86")] - use core::arch::x86 as arch; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64 as arch; - - fn adc(c_in: u8, a: u32, b: u32) -> (u8, u32) { - let mut sum = 0; - // SAFETY: There are no safety requirements for calling `_addcarry_u32`. - // It's just unsafe for API consistency with other intrinsics. - let c_out = unsafe { arch::_addcarry_u32(c_in, a, b, &mut sum) }; - (c_out, sum) - } - - fn sbb(b_in: u8, a: u32, b: u32) -> (u8, u32) { - let mut sum = 0; - // SAFETY: There are no safety requirements for calling `_subborrow_u32`. - // It's just unsafe for API consistency with other intrinsics. - let b_out = unsafe { arch::_subborrow_u32(b_in, a, b, &mut sum) }; - (b_out, sum) - } - - pub fn main() { - assert_eq!(adc(0, 1, 1), (0, 2)); - assert_eq!(adc(1, 1, 1), (0, 3)); - assert_eq!(adc(2, 1, 1), (0, 3)); // any non-zero carry acts as 1! - assert_eq!(adc(u8::MAX, 1, 1), (0, 3)); - assert_eq!(adc(0, u32::MAX, u32::MAX), (1, u32::MAX - 1)); - assert_eq!(adc(1, u32::MAX, u32::MAX), (1, u32::MAX)); - assert_eq!(adc(2, u32::MAX, u32::MAX), (1, u32::MAX)); - assert_eq!(adc(u8::MAX, u32::MAX, u32::MAX), (1, u32::MAX)); - - assert_eq!(sbb(0, 1, 1), (0, 0)); - assert_eq!(sbb(1, 1, 1), (1, u32::MAX)); - assert_eq!(sbb(2, 1, 1), (1, u32::MAX)); // any non-zero borrow acts as 1! - assert_eq!(sbb(u8::MAX, 1, 1), (1, u32::MAX)); - assert_eq!(sbb(0, 2, 1), (0, 1)); - assert_eq!(sbb(1, 2, 1), (0, 0)); - assert_eq!(sbb(2, 2, 1), (0, 0)); - assert_eq!(sbb(u8::MAX, 2, 1), (0, 0)); - assert_eq!(sbb(0, 1, 2), (1, u32::MAX)); - assert_eq!(sbb(1, 1, 2), (1, u32::MAX - 1)); - assert_eq!(sbb(2, 1, 2), (1, u32::MAX - 1)); - assert_eq!(sbb(u8::MAX, 1, 2), (1, u32::MAX - 1)); - } -} - -#[cfg(target_arch = "x86_64")] -mod x86_64 { - use core::arch::x86_64 as arch; - - fn adc(c_in: u8, a: u64, b: u64) -> (u8, u64) { - let mut sum = 0; - // SAFETY: There are no safety requirements for calling `_addcarry_u64`. - // It's just unsafe for API consistency with other intrinsics. - let c_out = unsafe { arch::_addcarry_u64(c_in, a, b, &mut sum) }; - (c_out, sum) - } - - fn sbb(b_in: u8, a: u64, b: u64) -> (u8, u64) { - let mut sum = 0; - // SAFETY: There are no safety requirements for calling `_subborrow_u64`. - // It's just unsafe for API consistency with other intrinsics. - let b_out = unsafe { arch::_subborrow_u64(b_in, a, b, &mut sum) }; - (b_out, sum) - } - - pub fn main() { - assert_eq!(adc(0, 1, 1), (0, 2)); - assert_eq!(adc(1, 1, 1), (0, 3)); - assert_eq!(adc(2, 1, 1), (0, 3)); // any non-zero carry acts as 1! - assert_eq!(adc(u8::MAX, 1, 1), (0, 3)); - assert_eq!(adc(0, u64::MAX, u64::MAX), (1, u64::MAX - 1)); - assert_eq!(adc(1, u64::MAX, u64::MAX), (1, u64::MAX)); - assert_eq!(adc(2, u64::MAX, u64::MAX), (1, u64::MAX)); - assert_eq!(adc(u8::MAX, u64::MAX, u64::MAX), (1, u64::MAX)); - - assert_eq!(sbb(0, 1, 1), (0, 0)); - assert_eq!(sbb(1, 1, 1), (1, u64::MAX)); - assert_eq!(sbb(2, 1, 1), (1, u64::MAX)); // any non-zero borrow acts as 1! - assert_eq!(sbb(u8::MAX, 1, 1), (1, u64::MAX)); - assert_eq!(sbb(0, 2, 1), (0, 1)); - assert_eq!(sbb(1, 2, 1), (0, 0)); - assert_eq!(sbb(2, 2, 1), (0, 0)); - assert_eq!(sbb(u8::MAX, 2, 1), (0, 0)); - assert_eq!(sbb(0, 1, 2), (1, u64::MAX)); - assert_eq!(sbb(1, 1, 2), (1, u64::MAX - 1)); - assert_eq!(sbb(2, 1, 2), (1, u64::MAX - 1)); - assert_eq!(sbb(u8::MAX, 1, 2), (1, u64::MAX - 1)); - } -} - -fn main() { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - x86::main(); - #[cfg(target_arch = "x86_64")] - x86_64::main(); -} diff --git a/src/tools/miri/tests/pass/intrinsics.rs b/src/tools/miri/tests/pass/intrinsics.rs deleted file mode 100644 index 0dda5aadce2..00000000000 --- a/src/tools/miri/tests/pass/intrinsics.rs +++ /dev/null @@ -1,60 +0,0 @@ -//@compile-flags: -Zmiri-permissive-provenance -#![feature(core_intrinsics, layout_for_ptr)] -//! Tests for various intrinsics that do not fit anywhere else. - -use std::intrinsics; -use std::mem::{discriminant, size_of, size_of_val, size_of_val_raw}; - -struct Bomb; - -impl Drop for Bomb { - fn drop(&mut self) { - eprintln!("BOOM!"); - } -} - -fn main() { - assert_eq!(size_of::>(), 8); - assert_eq!(size_of_val(&()), 0); - assert_eq!(size_of_val(&42), 4); - assert_eq!(size_of_val(&[] as &[i32]), 0); - assert_eq!(size_of_val(&[1, 2, 3] as &[i32]), 12); - assert_eq!(size_of_val("foobar"), 6); - - unsafe { - assert_eq!(size_of_val_raw(&[1] as &[i32] as *const [i32]), 4); - } - unsafe { - assert_eq!(size_of_val_raw(0x100 as *const i32), 4); - } - - assert_eq!(intrinsics::type_name::>(), "core::option::Option"); - - assert_eq!(intrinsics::likely(false), false); - assert_eq!(intrinsics::unlikely(true), true); - - let mut saw_true = false; - let mut saw_false = false; - - for _ in 0..50 { - if intrinsics::is_val_statically_known(0) { - saw_true = true; - } else { - saw_false = true; - } - } - assert!( - saw_true && saw_false, - "`is_val_statically_known` failed to return both true and false. Congrats, you won the lottery!" - ); - - intrinsics::forget(Bomb); - - let _v = intrinsics::discriminant_value(&Some(())); - let _v = intrinsics::discriminant_value(&0); - let _v = intrinsics::discriminant_value(&true); - let _v = intrinsics::discriminant_value(&vec![1, 2, 3]); - // Make sure that even if the discriminant is stored together with data, the intrinsic returns - // only the discriminant, nothing about the data. - assert_eq!(discriminant(&Some(false)), discriminant(&Some(true))); -} diff --git a/src/tools/miri/tests/pass/intrinsics/integer.rs b/src/tools/miri/tests/pass/intrinsics/integer.rs new file mode 100644 index 00000000000..13e7bd8e1b9 --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics/integer.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-FileCopyrightText: The Rust Project Developers (see https://thanks.rust-lang.org) + +#![feature(core_intrinsics)] +use std::intrinsics::*; + +pub fn main() { + unsafe { + [assert_eq!(ctpop(0u8), 0), assert_eq!(ctpop(0i8), 0)]; + [assert_eq!(ctpop(0u16), 0), assert_eq!(ctpop(0i16), 0)]; + [assert_eq!(ctpop(0u32), 0), assert_eq!(ctpop(0i32), 0)]; + [assert_eq!(ctpop(0u64), 0), assert_eq!(ctpop(0i64), 0)]; + + [assert_eq!(ctpop(1u8), 1), assert_eq!(ctpop(1i8), 1)]; + [assert_eq!(ctpop(1u16), 1), assert_eq!(ctpop(1i16), 1)]; + [assert_eq!(ctpop(1u32), 1), assert_eq!(ctpop(1i32), 1)]; + [assert_eq!(ctpop(1u64), 1), assert_eq!(ctpop(1i64), 1)]; + + [assert_eq!(ctpop(10u8), 2), assert_eq!(ctpop(10i8), 2)]; + [assert_eq!(ctpop(10u16), 2), assert_eq!(ctpop(10i16), 2)]; + [assert_eq!(ctpop(10u32), 2), assert_eq!(ctpop(10i32), 2)]; + [assert_eq!(ctpop(10u64), 2), assert_eq!(ctpop(10i64), 2)]; + + [assert_eq!(ctpop(100u8), 3), assert_eq!(ctpop(100i8), 3)]; + [assert_eq!(ctpop(100u16), 3), assert_eq!(ctpop(100i16), 3)]; + [assert_eq!(ctpop(100u32), 3), assert_eq!(ctpop(100i32), 3)]; + [assert_eq!(ctpop(100u64), 3), assert_eq!(ctpop(100i64), 3)]; + + [assert_eq!(ctpop(-1i8 as u8), 8), assert_eq!(ctpop(-1i8), 8)]; + [assert_eq!(ctpop(-1i16 as u16), 16), assert_eq!(ctpop(-1i16), 16)]; + [assert_eq!(ctpop(-1i32 as u32), 32), assert_eq!(ctpop(-1i32), 32)]; + [assert_eq!(ctpop(-1i64 as u64), 64), assert_eq!(ctpop(-1i64), 64)]; + + [assert_eq!(ctlz(0u8), 8), assert_eq!(ctlz(0i8), 8)]; + [assert_eq!(ctlz(0u16), 16), assert_eq!(ctlz(0i16), 16)]; + [assert_eq!(ctlz(0u32), 32), assert_eq!(ctlz(0i32), 32)]; + [assert_eq!(ctlz(0u64), 64), assert_eq!(ctlz(0i64), 64)]; + + [assert_eq!(ctlz(1u8), 7), assert_eq!(ctlz(1i8), 7)]; + [assert_eq!(ctlz(1u16), 15), assert_eq!(ctlz(1i16), 15)]; + [assert_eq!(ctlz(1u32), 31), assert_eq!(ctlz(1i32), 31)]; + [assert_eq!(ctlz(1u64), 63), assert_eq!(ctlz(1i64), 63)]; + + [assert_eq!(ctlz(10u8), 4), assert_eq!(ctlz(10i8), 4)]; + [assert_eq!(ctlz(10u16), 12), assert_eq!(ctlz(10i16), 12)]; + [assert_eq!(ctlz(10u32), 28), assert_eq!(ctlz(10i32), 28)]; + [assert_eq!(ctlz(10u64), 60), assert_eq!(ctlz(10i64), 60)]; + + [assert_eq!(ctlz(100u8), 1), assert_eq!(ctlz(100i8), 1)]; + [assert_eq!(ctlz(100u16), 9), assert_eq!(ctlz(100i16), 9)]; + [assert_eq!(ctlz(100u32), 25), assert_eq!(ctlz(100i32), 25)]; + [assert_eq!(ctlz(100u64), 57), assert_eq!(ctlz(100i64), 57)]; + + [assert_eq!(ctlz_nonzero(1u8), 7), assert_eq!(ctlz_nonzero(1i8), 7)]; + [assert_eq!(ctlz_nonzero(1u16), 15), assert_eq!(ctlz_nonzero(1i16), 15)]; + [assert_eq!(ctlz_nonzero(1u32), 31), assert_eq!(ctlz_nonzero(1i32), 31)]; + [assert_eq!(ctlz_nonzero(1u64), 63), assert_eq!(ctlz_nonzero(1i64), 63)]; + + [assert_eq!(ctlz_nonzero(10u8), 4), assert_eq!(ctlz_nonzero(10i8), 4)]; + [assert_eq!(ctlz_nonzero(10u16), 12), assert_eq!(ctlz_nonzero(10i16), 12)]; + [assert_eq!(ctlz_nonzero(10u32), 28), assert_eq!(ctlz_nonzero(10i32), 28)]; + [assert_eq!(ctlz_nonzero(10u64), 60), assert_eq!(ctlz_nonzero(10i64), 60)]; + + [assert_eq!(ctlz_nonzero(100u8), 1), assert_eq!(ctlz_nonzero(100i8), 1)]; + [assert_eq!(ctlz_nonzero(100u16), 9), assert_eq!(ctlz_nonzero(100i16), 9)]; + [assert_eq!(ctlz_nonzero(100u32), 25), assert_eq!(ctlz_nonzero(100i32), 25)]; + [assert_eq!(ctlz_nonzero(100u64), 57), assert_eq!(ctlz_nonzero(100i64), 57)]; + + [assert_eq!(cttz(-1i8 as u8), 0), assert_eq!(cttz(-1i8), 0)]; + [assert_eq!(cttz(-1i16 as u16), 0), assert_eq!(cttz(-1i16), 0)]; + [assert_eq!(cttz(-1i32 as u32), 0), assert_eq!(cttz(-1i32), 0)]; + [assert_eq!(cttz(-1i64 as u64), 0), assert_eq!(cttz(-1i64), 0)]; + + [assert_eq!(cttz(0u8), 8), assert_eq!(cttz(0i8), 8)]; + [assert_eq!(cttz(0u16), 16), assert_eq!(cttz(0i16), 16)]; + [assert_eq!(cttz(0u32), 32), assert_eq!(cttz(0i32), 32)]; + [assert_eq!(cttz(0u64), 64), assert_eq!(cttz(0i64), 64)]; + + [assert_eq!(cttz(1u8), 0), assert_eq!(cttz(1i8), 0)]; + [assert_eq!(cttz(1u16), 0), assert_eq!(cttz(1i16), 0)]; + [assert_eq!(cttz(1u32), 0), assert_eq!(cttz(1i32), 0)]; + [assert_eq!(cttz(1u64), 0), assert_eq!(cttz(1i64), 0)]; + + [assert_eq!(cttz(10u8), 1), assert_eq!(cttz(10i8), 1)]; + [assert_eq!(cttz(10u16), 1), assert_eq!(cttz(10i16), 1)]; + [assert_eq!(cttz(10u32), 1), assert_eq!(cttz(10i32), 1)]; + [assert_eq!(cttz(10u64), 1), assert_eq!(cttz(10i64), 1)]; + + [assert_eq!(cttz(100u8), 2), assert_eq!(cttz(100i8), 2)]; + [assert_eq!(cttz(100u16), 2), assert_eq!(cttz(100i16), 2)]; + [assert_eq!(cttz(100u32), 2), assert_eq!(cttz(100i32), 2)]; + [assert_eq!(cttz(100u64), 2), assert_eq!(cttz(100i64), 2)]; + + [assert_eq!(cttz_nonzero(-1i8 as u8), 0), assert_eq!(cttz_nonzero(-1i8), 0)]; + [assert_eq!(cttz_nonzero(-1i16 as u16), 0), assert_eq!(cttz_nonzero(-1i16), 0)]; + [assert_eq!(cttz_nonzero(-1i32 as u32), 0), assert_eq!(cttz_nonzero(-1i32), 0)]; + [assert_eq!(cttz_nonzero(-1i64 as u64), 0), assert_eq!(cttz_nonzero(-1i64), 0)]; + + [assert_eq!(cttz_nonzero(1u8), 0), assert_eq!(cttz_nonzero(1i8), 0)]; + [assert_eq!(cttz_nonzero(1u16), 0), assert_eq!(cttz_nonzero(1i16), 0)]; + [assert_eq!(cttz_nonzero(1u32), 0), assert_eq!(cttz_nonzero(1i32), 0)]; + [assert_eq!(cttz_nonzero(1u64), 0), assert_eq!(cttz_nonzero(1i64), 0)]; + + [assert_eq!(cttz_nonzero(10u8), 1), assert_eq!(cttz_nonzero(10i8), 1)]; + [assert_eq!(cttz_nonzero(10u16), 1), assert_eq!(cttz_nonzero(10i16), 1)]; + [assert_eq!(cttz_nonzero(10u32), 1), assert_eq!(cttz_nonzero(10i32), 1)]; + [assert_eq!(cttz_nonzero(10u64), 1), assert_eq!(cttz_nonzero(10i64), 1)]; + + [assert_eq!(cttz_nonzero(100u8), 2), assert_eq!(cttz_nonzero(100i8), 2)]; + [assert_eq!(cttz_nonzero(100u16), 2), assert_eq!(cttz_nonzero(100i16), 2)]; + [assert_eq!(cttz_nonzero(100u32), 2), assert_eq!(cttz_nonzero(100i32), 2)]; + [assert_eq!(cttz_nonzero(100u64), 2), assert_eq!(cttz_nonzero(100i64), 2)]; + + assert_eq!(bswap(0x0Au8), 0x0A); // no-op + assert_eq!(bswap(0x0Ai8), 0x0A); // no-op + assert_eq!(bswap(0x0A0Bu16), 0x0B0A); + assert_eq!(bswap(0x0A0Bi16), 0x0B0A); + assert_eq!(bswap(0x0ABBCC0Du32), 0x0DCCBB0A); + assert_eq!(bswap(0x0ABBCC0Di32), 0x0DCCBB0A); + assert_eq!(bswap(0x0122334455667708u64), 0x0877665544332201); + assert_eq!(bswap(0x0122334455667708i64), 0x0877665544332201); + + assert_eq!(exact_div(9 * 9u32, 3), 27); + assert_eq!(exact_div(-9 * 9i32, 3), -27); + assert_eq!(exact_div(9 * 9i8, -3), -27); + assert_eq!(exact_div(-9 * 9i64, -3), 27); + + assert_eq!(unchecked_div(9 * 9u32, 2), 40); + assert_eq!(unchecked_div(-9 * 9i32, 2), -40); + assert_eq!(unchecked_div(9 * 9i8, -2), -40); + assert_eq!(unchecked_div(-9 * 9i64, -2), 40); + + assert_eq!(unchecked_rem(9 * 9u32, 2), 1); + assert_eq!(unchecked_rem(-9 * 9i32, 2), -1); + assert_eq!(unchecked_rem(9 * 9i8, -2), 1); + assert_eq!(unchecked_rem(-9 * 9i64, -2), -1); + + assert_eq!(unchecked_add(23u8, 19), 42); + assert_eq!(unchecked_add(5, -10), -5); + + assert_eq!(unchecked_sub(23u8, 19), 4); + assert_eq!(unchecked_sub(-17, -27), 10); + + assert_eq!(unchecked_mul(6u8, 7), 42); + assert_eq!(unchecked_mul(13, -5), -65); + } +} diff --git a/src/tools/miri/tests/pass/intrinsics/intrinsics.rs b/src/tools/miri/tests/pass/intrinsics/intrinsics.rs new file mode 100644 index 00000000000..0dda5aadce2 --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics/intrinsics.rs @@ -0,0 +1,60 @@ +//@compile-flags: -Zmiri-permissive-provenance +#![feature(core_intrinsics, layout_for_ptr)] +//! Tests for various intrinsics that do not fit anywhere else. + +use std::intrinsics; +use std::mem::{discriminant, size_of, size_of_val, size_of_val_raw}; + +struct Bomb; + +impl Drop for Bomb { + fn drop(&mut self) { + eprintln!("BOOM!"); + } +} + +fn main() { + assert_eq!(size_of::>(), 8); + assert_eq!(size_of_val(&()), 0); + assert_eq!(size_of_val(&42), 4); + assert_eq!(size_of_val(&[] as &[i32]), 0); + assert_eq!(size_of_val(&[1, 2, 3] as &[i32]), 12); + assert_eq!(size_of_val("foobar"), 6); + + unsafe { + assert_eq!(size_of_val_raw(&[1] as &[i32] as *const [i32]), 4); + } + unsafe { + assert_eq!(size_of_val_raw(0x100 as *const i32), 4); + } + + assert_eq!(intrinsics::type_name::>(), "core::option::Option"); + + assert_eq!(intrinsics::likely(false), false); + assert_eq!(intrinsics::unlikely(true), true); + + let mut saw_true = false; + let mut saw_false = false; + + for _ in 0..50 { + if intrinsics::is_val_statically_known(0) { + saw_true = true; + } else { + saw_false = true; + } + } + assert!( + saw_true && saw_false, + "`is_val_statically_known` failed to return both true and false. Congrats, you won the lottery!" + ); + + intrinsics::forget(Bomb); + + let _v = intrinsics::discriminant_value(&Some(())); + let _v = intrinsics::discriminant_value(&0); + let _v = intrinsics::discriminant_value(&true); + let _v = intrinsics::discriminant_value(&vec![1, 2, 3]); + // Make sure that even if the discriminant is stored together with data, the intrinsic returns + // only the discriminant, nothing about the data. + assert_eq!(discriminant(&Some(false)), discriminant(&Some(true))); +} diff --git a/src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs new file mode 100644 index 00000000000..096ec78da1a --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics/portable-simd-ptrs.rs @@ -0,0 +1,12 @@ +// Separate test without strict provenance +//@compile-flags: -Zmiri-permissive-provenance +#![feature(portable_simd)] +use std::ptr; +use std::simd::prelude::*; + +fn main() { + // Pointer casts + let _val: Simd<*const u8, 4> = Simd::<*const i32, 4>::splat(ptr::null()).cast(); + let addrs = Simd::<*const i32, 4>::splat(ptr::null()).expose_provenance(); + let _ptrs = Simd::<*const i32, 4>::with_exposed_provenance(addrs); +} diff --git a/src/tools/miri/tests/pass/intrinsics/portable-simd.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs new file mode 100644 index 00000000000..1fc713d48dc --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs @@ -0,0 +1,581 @@ +//@compile-flags: -Zmiri-strict-provenance +#![feature(portable_simd, adt_const_params, core_intrinsics)] +#![allow(incomplete_features, internal_features)] +use std::intrinsics::simd as intrinsics; +use std::ptr; +use std::simd::{prelude::*, StdFloat}; + +fn simd_ops_f32() { + let a = f32x4::splat(10.0); + let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); + assert_eq!(-b, f32x4::from_array([-1.0, -2.0, -3.0, 4.0])); + assert_eq!(a + b, f32x4::from_array([11.0, 12.0, 13.0, 6.0])); + assert_eq!(a - b, f32x4::from_array([9.0, 8.0, 7.0, 14.0])); + assert_eq!(a * b, f32x4::from_array([10.0, 20.0, 30.0, -40.0])); + assert_eq!(b / a, f32x4::from_array([0.1, 0.2, 0.3, -0.4])); + assert_eq!(a / f32x4::splat(2.0), f32x4::splat(5.0)); + assert_eq!(a % b, f32x4::from_array([0.0, 0.0, 1.0, 2.0])); + assert_eq!(b.abs(), f32x4::from_array([1.0, 2.0, 3.0, 4.0])); + assert_eq!(a.simd_max(b * f32x4::splat(4.0)), f32x4::from_array([10.0, 10.0, 12.0, 10.0])); + assert_eq!(a.simd_min(b * f32x4::splat(4.0)), f32x4::from_array([4.0, 8.0, 10.0, -16.0])); + + assert_eq!(a.mul_add(b, a), (a * b) + a); + assert_eq!(b.mul_add(b, a), (b * b) + a); + assert_eq!(a.mul_add(b, b), (a * b) + b); + assert_eq!( + f32x4::splat(-3.2).mul_add(b, f32x4::splat(f32::NEG_INFINITY)), + f32x4::splat(f32::NEG_INFINITY) + ); + assert_eq!((a * a).sqrt(), a); + assert_eq!((b * b).sqrt(), b.abs()); + + assert_eq!(a.simd_eq(f32x4::splat(5.0) * b), Mask::from_array([false, true, false, false])); + assert_eq!(a.simd_ne(f32x4::splat(5.0) * b), Mask::from_array([true, false, true, true])); + assert_eq!(a.simd_le(f32x4::splat(5.0) * b), Mask::from_array([false, true, true, false])); + assert_eq!(a.simd_lt(f32x4::splat(5.0) * b), Mask::from_array([false, false, true, false])); + assert_eq!(a.simd_ge(f32x4::splat(5.0) * b), Mask::from_array([true, true, false, true])); + assert_eq!(a.simd_gt(f32x4::splat(5.0) * b), Mask::from_array([true, false, false, true])); + + assert_eq!(a.reduce_sum(), 40.0); + assert_eq!(b.reduce_sum(), 2.0); + assert_eq!(a.reduce_product(), 100.0 * 100.0); + assert_eq!(b.reduce_product(), -24.0); + assert_eq!(a.reduce_max(), 10.0); + assert_eq!(b.reduce_max(), 3.0); + assert_eq!(a.reduce_min(), 10.0); + assert_eq!(b.reduce_min(), -4.0); + + assert_eq!( + f32x2::from_array([0.0, f32::NAN]).simd_max(f32x2::from_array([f32::NAN, 0.0])), + f32x2::from_array([0.0, 0.0]) + ); + assert_eq!(f32x2::from_array([0.0, f32::NAN]).reduce_max(), 0.0); + assert_eq!(f32x2::from_array([f32::NAN, 0.0]).reduce_max(), 0.0); + assert_eq!( + f32x2::from_array([0.0, f32::NAN]).simd_min(f32x2::from_array([f32::NAN, 0.0])), + f32x2::from_array([0.0, 0.0]) + ); + assert_eq!(f32x2::from_array([0.0, f32::NAN]).reduce_min(), 0.0); + assert_eq!(f32x2::from_array([f32::NAN, 0.0]).reduce_min(), 0.0); +} + +fn simd_ops_f64() { + let a = f64x4::splat(10.0); + let b = f64x4::from_array([1.0, 2.0, 3.0, -4.0]); + assert_eq!(-b, f64x4::from_array([-1.0, -2.0, -3.0, 4.0])); + assert_eq!(a + b, f64x4::from_array([11.0, 12.0, 13.0, 6.0])); + assert_eq!(a - b, f64x4::from_array([9.0, 8.0, 7.0, 14.0])); + assert_eq!(a * b, f64x4::from_array([10.0, 20.0, 30.0, -40.0])); + assert_eq!(b / a, f64x4::from_array([0.1, 0.2, 0.3, -0.4])); + assert_eq!(a / f64x4::splat(2.0), f64x4::splat(5.0)); + assert_eq!(a % b, f64x4::from_array([0.0, 0.0, 1.0, 2.0])); + assert_eq!(b.abs(), f64x4::from_array([1.0, 2.0, 3.0, 4.0])); + assert_eq!(a.simd_max(b * f64x4::splat(4.0)), f64x4::from_array([10.0, 10.0, 12.0, 10.0])); + assert_eq!(a.simd_min(b * f64x4::splat(4.0)), f64x4::from_array([4.0, 8.0, 10.0, -16.0])); + + assert_eq!(a.mul_add(b, a), (a * b) + a); + assert_eq!(b.mul_add(b, a), (b * b) + a); + assert_eq!(a.mul_add(b, b), (a * b) + b); + assert_eq!( + f64x4::splat(-3.2).mul_add(b, f64x4::splat(f64::NEG_INFINITY)), + f64x4::splat(f64::NEG_INFINITY) + ); + assert_eq!((a * a).sqrt(), a); + assert_eq!((b * b).sqrt(), b.abs()); + + assert_eq!(a.simd_eq(f64x4::splat(5.0) * b), Mask::from_array([false, true, false, false])); + assert_eq!(a.simd_ne(f64x4::splat(5.0) * b), Mask::from_array([true, false, true, true])); + assert_eq!(a.simd_le(f64x4::splat(5.0) * b), Mask::from_array([false, true, true, false])); + assert_eq!(a.simd_lt(f64x4::splat(5.0) * b), Mask::from_array([false, false, true, false])); + assert_eq!(a.simd_ge(f64x4::splat(5.0) * b), Mask::from_array([true, true, false, true])); + assert_eq!(a.simd_gt(f64x4::splat(5.0) * b), Mask::from_array([true, false, false, true])); + + assert_eq!(a.reduce_sum(), 40.0); + assert_eq!(b.reduce_sum(), 2.0); + assert_eq!(a.reduce_product(), 100.0 * 100.0); + assert_eq!(b.reduce_product(), -24.0); + assert_eq!(a.reduce_max(), 10.0); + assert_eq!(b.reduce_max(), 3.0); + assert_eq!(a.reduce_min(), 10.0); + assert_eq!(b.reduce_min(), -4.0); + + assert_eq!( + f64x2::from_array([0.0, f64::NAN]).simd_max(f64x2::from_array([f64::NAN, 0.0])), + f64x2::from_array([0.0, 0.0]) + ); + assert_eq!(f64x2::from_array([0.0, f64::NAN]).reduce_max(), 0.0); + assert_eq!(f64x2::from_array([f64::NAN, 0.0]).reduce_max(), 0.0); + assert_eq!( + f64x2::from_array([0.0, f64::NAN]).simd_min(f64x2::from_array([f64::NAN, 0.0])), + f64x2::from_array([0.0, 0.0]) + ); + assert_eq!(f64x2::from_array([0.0, f64::NAN]).reduce_min(), 0.0); + assert_eq!(f64x2::from_array([f64::NAN, 0.0]).reduce_min(), 0.0); +} + +fn simd_ops_i32() { + let a = i32x4::splat(10); + let b = i32x4::from_array([1, 2, 3, -4]); + assert_eq!(-b, i32x4::from_array([-1, -2, -3, 4])); + assert_eq!(a + b, i32x4::from_array([11, 12, 13, 6])); + assert_eq!(a - b, i32x4::from_array([9, 8, 7, 14])); + assert_eq!(a * b, i32x4::from_array([10, 20, 30, -40])); + assert_eq!(a / b, i32x4::from_array([10, 5, 3, -2])); + assert_eq!(a / i32x4::splat(2), i32x4::splat(5)); + assert_eq!(i32x2::splat(i32::MIN) / i32x2::splat(-1), i32x2::splat(i32::MIN)); + assert_eq!(a % b, i32x4::from_array([0, 0, 1, 2])); + assert_eq!(i32x2::splat(i32::MIN) % i32x2::splat(-1), i32x2::splat(0)); + assert_eq!(b.abs(), i32x4::from_array([1, 2, 3, 4])); + assert_eq!(a.simd_max(b * i32x4::splat(4)), i32x4::from_array([10, 10, 12, 10])); + assert_eq!(a.simd_min(b * i32x4::splat(4)), i32x4::from_array([4, 8, 10, -16])); + + assert_eq!( + i8x4::from_array([i8::MAX, -23, 23, i8::MIN]).saturating_add(i8x4::from_array([ + 1, + i8::MIN, + i8::MAX, + 28 + ])), + i8x4::from_array([i8::MAX, i8::MIN, i8::MAX, -100]) + ); + assert_eq!( + i8x4::from_array([i8::MAX, -28, 27, 42]).saturating_sub(i8x4::from_array([ + 1, + i8::MAX, + i8::MAX, + -80 + ])), + i8x4::from_array([126, i8::MIN, -100, 122]) + ); + assert_eq!( + u8x4::from_array([u8::MAX, 0, 23, 42]).saturating_add(u8x4::from_array([ + 1, + 1, + u8::MAX, + 200 + ])), + u8x4::from_array([u8::MAX, 1, u8::MAX, 242]) + ); + assert_eq!( + u8x4::from_array([u8::MAX, 0, 23, 42]).saturating_sub(u8x4::from_array([ + 1, + 1, + u8::MAX, + 200 + ])), + u8x4::from_array([254, 0, 0, 0]) + ); + + assert_eq!(!b, i32x4::from_array([!1, !2, !3, !-4])); + assert_eq!(b << i32x4::splat(2), i32x4::from_array([4, 8, 12, -16])); + assert_eq!(b >> i32x4::splat(1), i32x4::from_array([0, 1, 1, -2])); + assert_eq!(b & i32x4::splat(2), i32x4::from_array([0, 2, 2, 0])); + assert_eq!(b | i32x4::splat(2), i32x4::from_array([3, 2, 3, -2])); + assert_eq!(b ^ i32x4::splat(2), i32x4::from_array([3, 0, 1, -2])); + + assert_eq!(a.simd_eq(i32x4::splat(5) * b), Mask::from_array([false, true, false, false])); + assert_eq!(a.simd_ne(i32x4::splat(5) * b), Mask::from_array([true, false, true, true])); + assert_eq!(a.simd_le(i32x4::splat(5) * b), Mask::from_array([false, true, true, false])); + assert_eq!(a.simd_lt(i32x4::splat(5) * b), Mask::from_array([false, false, true, false])); + assert_eq!(a.simd_ge(i32x4::splat(5) * b), Mask::from_array([true, true, false, true])); + assert_eq!(a.simd_gt(i32x4::splat(5) * b), Mask::from_array([true, false, false, true])); + + assert_eq!(a.reduce_sum(), 40); + assert_eq!(b.reduce_sum(), 2); + assert_eq!(a.reduce_product(), 100 * 100); + assert_eq!(b.reduce_product(), -24); + assert_eq!(a.reduce_max(), 10); + assert_eq!(b.reduce_max(), 3); + assert_eq!(a.reduce_min(), 10); + assert_eq!(b.reduce_min(), -4); + + assert_eq!(a.reduce_and(), 10); + assert_eq!(b.reduce_and(), 0); + assert_eq!(a.reduce_or(), 10); + assert_eq!(b.reduce_or(), -1); + assert_eq!(a.reduce_xor(), 0); + assert_eq!(b.reduce_xor(), -4); + + assert_eq!(b.leading_zeros(), u32x4::from_array([31, 30, 30, 0])); + assert_eq!(b.trailing_zeros(), u32x4::from_array([0, 1, 0, 2])); + assert_eq!(b.leading_ones(), u32x4::from_array([0, 0, 0, 30])); + assert_eq!(b.trailing_ones(), u32x4::from_array([1, 0, 2, 0])); + assert_eq!( + b.swap_bytes(), + i32x4::from_array([0x01000000, 0x02000000, 0x03000000, 0xfcffffffu32 as i32]) + ); + assert_eq!( + b.reverse_bits(), + i32x4::from_array([ + 0x80000000u32 as i32, + 0x40000000, + 0xc0000000u32 as i32, + 0x3fffffffu32 as i32 + ]) + ); +} + +fn simd_mask() { + use std::intrinsics::simd::*; + + let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0])); + assert_eq!(intmask, Mask::from_array([false, true, false, false])); + assert_eq!(intmask.to_array(), [false, true, false, false]); + + let values = [ + true, false, false, true, false, false, true, false, true, true, false, false, false, true, + false, true, + ]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1010001101001001); + assert_eq!(Mask::::from_bitmask(bitmask), mask); + + // Also directly call intrinsic, to test both kinds of return types. + unsafe { + let bitmask1: u16 = simd_bitmask(mask.to_int()); + let bitmask2: [u8; 2] = simd_bitmask(mask.to_int()); + if cfg!(target_endian = "little") { + assert_eq!(bitmask1, 0b1010001101001001); + assert_eq!(bitmask2, [0b01001001, 0b10100011]); + } else { + // All the bitstrings are reversed compared to above, but the array elements are in the + // same order. + assert_eq!(bitmask1, 0b1001001011000101); + assert_eq!(bitmask2, [0b10010010, 0b11000101]); + } + } + + // Mask less than 8 bits long, which is a special case (padding with 0s). + let values = [false, false, false, true]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000); + assert_eq!(Mask::::from_bitmask(bitmask), mask); + unsafe { + let bitmask1: u8 = simd_bitmask(mask.to_int()); + let bitmask2: [u8; 1] = simd_bitmask(mask.to_int()); + if cfg!(target_endian = "little") { + assert_eq!(bitmask1, 0b1000); + assert_eq!(bitmask2, [0b1000]); + } else { + assert_eq!(bitmask1, 0b0001); + assert_eq!(bitmask2, [0b0001]); + } + } + + // This used to cause an ICE. It exercises simd_select_bitmask with an array as input. + let bitmask = u8x4::from_array([0b00001101, 0, 0, 0]); + assert_eq!( + mask32x4::from_bitmask_vector(bitmask), + mask32x4::from_array([true, false, true, true]), + ); + let bitmask = u8x8::from_array([0b01000101, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + mask32x8::from_bitmask_vector(bitmask), + mask32x8::from_array([true, false, true, false, false, false, true, false]), + ); + let bitmask = + u8x16::from_array([0b01000101, 0b11110000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + mask32x16::from_bitmask_vector(bitmask), + mask32x16::from_array([ + true, false, true, false, false, false, true, false, false, false, false, false, true, + true, true, true, + ]), + ); + + // Also directly call simd_select_bitmask, to test both kinds of argument types. + unsafe { + // These masks are exactly the results we got out above in the `simd_bitmask` tests. + let selected1 = simd_select_bitmask::( + if cfg!(target_endian = "little") { 0b1010001101001001 } else { 0b1001001011000101 }, + i32x16::splat(1), // yes + i32x16::splat(0), // no + ); + let selected2 = simd_select_bitmask::<[u8; 2], _>( + if cfg!(target_endian = "little") { + [0b01001001, 0b10100011] + } else { + [0b10010010, 0b11000101] + }, + i32x16::splat(1), // yes + i32x16::splat(0), // no + ); + assert_eq!(selected1, i32x16::from_array([1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1])); + assert_eq!(selected2, selected1); + // Also try masks less than a byte long. + let selected1 = simd_select_bitmask::( + if cfg!(target_endian = "little") { 0b1000 } else { 0b0001 }, + i32x4::splat(1), // yes + i32x4::splat(0), // no + ); + let selected2 = simd_select_bitmask::<[u8; 1], _>( + if cfg!(target_endian = "little") { [0b1000] } else { [0b0001] }, + i32x4::splat(1), // yes + i32x4::splat(0), // no + ); + assert_eq!(selected1, i32x4::from_array([0, 0, 0, 1])); + assert_eq!(selected2, selected1); + } +} + +fn simd_cast() { + // between integer types + assert_eq!(i32x4::from_array([1, 2, 3, -4]), i16x4::from_array([1, 2, 3, -4]).cast()); + assert_eq!(i16x4::from_array([1, 2, 3, -4]), i32x4::from_array([1, 2, 3, -4]).cast()); + assert_eq!(i32x4::from_array([1, -1, 3, 4]), u64x4::from_array([1, u64::MAX, 3, 4]).cast()); + + // float -> int + assert_eq!( + i8x4::from_array([127, -128, 127, -128]), + f32x4::from_array([127.99, -128.99, 999.0, -999.0]).cast() + ); + assert_eq!( + i32x4::from_array([0, 1, -1, 2147483520]), + f32x4::from_array([ + -0.0, + /*0x1.19999ap+0*/ f32::from_bits(0x3f8ccccd), + /*-0x1.19999ap+0*/ f32::from_bits(0xbf8ccccd), + 2147483520.0 + ]) + .cast() + ); + assert_eq!( + i32x8::from_array([i32::MAX, i32::MIN, i32::MAX, i32::MIN, i32::MAX, i32::MIN, 0, 0]), + f32x8::from_array([ + 2147483648.0f32, + -2147483904.0f32, + f32::MAX, + f32::MIN, + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, + -f32::NAN, + ]) + .cast() + ); + + // int -> float + assert_eq!( + f32x4::from_array([ + -2147483648.0, + /*0x1.26580cp+30*/ f32::from_bits(0x4e932c06), + 16777220.0, + -16777220.0, + ]), + i32x4::from_array([-2147483647i32, 1234567890i32, 16777219i32, -16777219i32]).cast() + ); + + // float -> float + assert_eq!( + f32x4::from_array([f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY]), + f64x4::from_array([f64::MAX, f64::INFINITY, f64::MIN, f64::NEG_INFINITY]).cast() + ); + + // unchecked casts + unsafe { + assert_eq!( + i32x4::from_array([0, 1, -1, 2147483520]), + f32x4::from_array([ + -0.0, + /*0x1.19999ap+0*/ f32::from_bits(0x3f8ccccd), + /*-0x1.19999ap+0*/ f32::from_bits(0xbf8ccccd), + 2147483520.0 + ]) + .to_int_unchecked() + ); + assert_eq!( + u64x4::from_array([0, 10000000000000000, u64::MAX - 2047, 9223372036854775808]), + f64x4::from_array([ + -0.99999999999, + 1e16, + (u64::MAX - 1024) as f64, + 9223372036854775808.0 + ]) + .to_int_unchecked() + ); + } +} + +fn simd_swizzle() { + let a = f32x4::splat(10.0); + let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); + + assert_eq!(simd_swizzle!(b, [3, 0, 0, 2]), f32x4::from_array([-4.0, 1.0, 1.0, 3.0])); + assert_eq!(simd_swizzle!(b, [1, 2]), f32x2::from_array([2.0, 3.0])); + assert_eq!(simd_swizzle!(b, a, [3, 4]), f32x2::from_array([-4.0, 10.0])); +} + +fn simd_gather_scatter() { + let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; + let idxs = Simd::from_array([9, 3, 0, 17]); + let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. + assert_eq!(result, Simd::from_array([0, 13, 10, 0])); + + let idxs = Simd::from_array([9, 3, 0, 0]); + Simd::from_array([-27, 82, -41, 124]).scatter(&mut vec, idxs); + assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); + + // We call the intrinsics directly to experiment with dangling pointers and masks. + let val = 42u8; + let ptrs: Simd<*const u8, 4> = + Simd::from_array([ptr::null(), ptr::addr_of!(val), ptr::addr_of!(val), ptr::addr_of!(val)]); + let default = u8x4::splat(0); + let mask = i8x4::from_array([0, !0, 0, !0]); + let vals = unsafe { intrinsics::simd_gather(default, ptrs, mask) }; + assert_eq!(vals, u8x4::from_array([0, 42, 0, 42]),); + + let mut val1 = 0u8; + let mut val2 = 0u8; + let ptrs: Simd<*mut u8, 4> = Simd::from_array([ + ptr::null_mut(), + ptr::addr_of_mut!(val1), + ptr::addr_of_mut!(val1), + ptr::addr_of_mut!(val2), + ]); + let vals = u8x4::from_array([1, 2, 3, 4]); + unsafe { intrinsics::simd_scatter(vals, ptrs, mask) }; + assert_eq!(val1, 2); + assert_eq!(val2, 4); + + // Also check what happens when `scatter` has multiple overlapping pointers. + let mut val = 0u8; + let ptrs: Simd<*mut u8, 4> = Simd::from_array([ + ptr::addr_of_mut!(val), + ptr::addr_of_mut!(val), + ptr::addr_of_mut!(val), + ptr::addr_of_mut!(val), + ]); + let vals = u8x4::from_array([1, 2, 3, 4]); + unsafe { intrinsics::simd_scatter(vals, ptrs, mask) }; + assert_eq!(val, 4); +} + +fn simd_round() { + assert_eq!( + f32x4::from_array([0.9, 1.001, 2.0, -4.5]).ceil(), + f32x4::from_array([1.0, 2.0, 2.0, -4.0]) + ); + assert_eq!( + f32x4::from_array([0.9, 1.001, 2.0, -4.5]).floor(), + f32x4::from_array([0.0, 1.0, 2.0, -5.0]) + ); + assert_eq!( + f32x4::from_array([0.9, 1.001, 2.0, -4.5]).round(), + f32x4::from_array([1.0, 1.0, 2.0, -5.0]) + ); + assert_eq!( + f32x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(), + f32x4::from_array([0.0, 1.0, 2.0, -4.0]) + ); + + assert_eq!( + f64x4::from_array([0.9, 1.001, 2.0, -4.5]).ceil(), + f64x4::from_array([1.0, 2.0, 2.0, -4.0]) + ); + assert_eq!( + f64x4::from_array([0.9, 1.001, 2.0, -4.5]).floor(), + f64x4::from_array([0.0, 1.0, 2.0, -5.0]) + ); + assert_eq!( + f64x4::from_array([0.9, 1.001, 2.0, -4.5]).round(), + f64x4::from_array([1.0, 1.0, 2.0, -5.0]) + ); + assert_eq!( + f64x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(), + f64x4::from_array([0.0, 1.0, 2.0, -4.0]) + ); +} + +fn simd_intrinsics() { + use intrinsics::*; + + unsafe { + // Make sure simd_eq returns all-1 for `true` + let a = i32x4::splat(10); + let b = i32x4::from_array([1, 2, 10, 4]); + let c: i32x4 = simd_eq(a, b); + assert_eq!(c, i32x4::from_array([0, 0, -1, 0])); + + assert!(!simd_reduce_any(i32x4::splat(0))); + assert!(simd_reduce_any(i32x4::splat(-1))); + assert!(simd_reduce_any(i32x2::from_array([0, -1]))); + assert!(!simd_reduce_all(i32x4::splat(0))); + assert!(simd_reduce_all(i32x4::splat(-1))); + assert!(!simd_reduce_all(i32x2::from_array([0, -1]))); + + assert_eq!( + simd_select(i8x4::from_array([0, -1, -1, 0]), a, b), + i32x4::from_array([1, 10, 10, 4]) + ); + assert_eq!( + simd_select(i8x4::from_array([0, -1, -1, 0]), b, a), + i32x4::from_array([10, 2, 10, 10]) + ); + assert_eq!(simd_shuffle_generic::<_, i32x4, { &[3, 1, 0, 2] }>(a, b), a,); + assert_eq!(simd_shuffle::<_, _, i32x4>(a, b, const { [3u32, 1, 0, 2] }), a,); + assert_eq!( + simd_shuffle_generic::<_, i32x4, { &[7, 5, 4, 6] }>(a, b), + i32x4::from_array([4, 2, 1, 10]), + ); + assert_eq!( + simd_shuffle::<_, _, i32x4>(a, b, const { [7u32, 5, 4, 6] }), + i32x4::from_array([4, 2, 1, 10]), + ); + } +} + +fn simd_float_intrinsics() { + use intrinsics::*; + + // These are just smoke tests to ensure the intrinsics can be called. + unsafe { + let a = f32x4::splat(10.0); + simd_fsqrt(a); + simd_fsin(a); + simd_fcos(a); + simd_fexp(a); + simd_fexp2(a); + simd_flog(a); + simd_flog2(a); + simd_flog10(a); + } +} + +fn simd_masked_loadstore() { + // The buffer is deliberarely too short, so reading the last element would be UB. + let buf = [3i32; 3]; + let default = i32x4::splat(0); + let mask = i32x4::from_array([!0, !0, !0, 0]); + let vals = unsafe { intrinsics::simd_masked_load(mask, buf.as_ptr(), default) }; + assert_eq!(vals, i32x4::from_array([3, 3, 3, 0])); + // Also read in a way that the *first* element is OOB. + let mask2 = i32x4::from_array([0, !0, !0, !0]); + let vals = + unsafe { intrinsics::simd_masked_load(mask2, buf.as_ptr().wrapping_sub(1), default) }; + assert_eq!(vals, i32x4::from_array([0, 3, 3, 3])); + + // The buffer is deliberarely too short, so writing the last element would be UB. + let mut buf = [42i32; 3]; + let vals = i32x4::from_array([1, 2, 3, 4]); + unsafe { intrinsics::simd_masked_store(mask, buf.as_mut_ptr(), vals) }; + assert_eq!(buf, [1, 2, 3]); + // Also write in a way that the *first* element is OOB. + unsafe { intrinsics::simd_masked_store(mask2, buf.as_mut_ptr().wrapping_sub(1), vals) }; + assert_eq!(buf, [2, 3, 4]); +} + +fn main() { + simd_mask(); + simd_ops_f32(); + simd_ops_f64(); + simd_ops_i32(); + simd_cast(); + simd_swizzle(); + simd_gather_scatter(); + simd_round(); + simd_intrinsics(); + simd_float_intrinsics(); + simd_masked_loadstore(); +} diff --git a/src/tools/miri/tests/pass/intrinsics/volatile.rs b/src/tools/miri/tests/pass/intrinsics/volatile.rs new file mode 100644 index 00000000000..c9799801455 --- /dev/null +++ b/src/tools/miri/tests/pass/intrinsics/volatile.rs @@ -0,0 +1,12 @@ +// related: #58645 +#![feature(core_intrinsics)] +use std::intrinsics::{volatile_load, volatile_store}; + +pub fn main() { + unsafe { + let i: &mut (isize, isize) = &mut (0, 0); + volatile_store(i, (1, 2)); + assert_eq!(volatile_load(i), (1, 2)); + assert_eq!(i, &mut (1, 2)); + } +} diff --git a/src/tools/miri/tests/pass/portable-simd-ptrs.rs b/src/tools/miri/tests/pass/portable-simd-ptrs.rs deleted file mode 100644 index 096ec78da1a..00000000000 --- a/src/tools/miri/tests/pass/portable-simd-ptrs.rs +++ /dev/null @@ -1,12 +0,0 @@ -// Separate test without strict provenance -//@compile-flags: -Zmiri-permissive-provenance -#![feature(portable_simd)] -use std::ptr; -use std::simd::prelude::*; - -fn main() { - // Pointer casts - let _val: Simd<*const u8, 4> = Simd::<*const i32, 4>::splat(ptr::null()).cast(); - let addrs = Simd::<*const i32, 4>::splat(ptr::null()).expose_provenance(); - let _ptrs = Simd::<*const i32, 4>::with_exposed_provenance(addrs); -} diff --git a/src/tools/miri/tests/pass/portable-simd.rs b/src/tools/miri/tests/pass/portable-simd.rs deleted file mode 100644 index 1fc713d48dc..00000000000 --- a/src/tools/miri/tests/pass/portable-simd.rs +++ /dev/null @@ -1,581 +0,0 @@ -//@compile-flags: -Zmiri-strict-provenance -#![feature(portable_simd, adt_const_params, core_intrinsics)] -#![allow(incomplete_features, internal_features)] -use std::intrinsics::simd as intrinsics; -use std::ptr; -use std::simd::{prelude::*, StdFloat}; - -fn simd_ops_f32() { - let a = f32x4::splat(10.0); - let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); - assert_eq!(-b, f32x4::from_array([-1.0, -2.0, -3.0, 4.0])); - assert_eq!(a + b, f32x4::from_array([11.0, 12.0, 13.0, 6.0])); - assert_eq!(a - b, f32x4::from_array([9.0, 8.0, 7.0, 14.0])); - assert_eq!(a * b, f32x4::from_array([10.0, 20.0, 30.0, -40.0])); - assert_eq!(b / a, f32x4::from_array([0.1, 0.2, 0.3, -0.4])); - assert_eq!(a / f32x4::splat(2.0), f32x4::splat(5.0)); - assert_eq!(a % b, f32x4::from_array([0.0, 0.0, 1.0, 2.0])); - assert_eq!(b.abs(), f32x4::from_array([1.0, 2.0, 3.0, 4.0])); - assert_eq!(a.simd_max(b * f32x4::splat(4.0)), f32x4::from_array([10.0, 10.0, 12.0, 10.0])); - assert_eq!(a.simd_min(b * f32x4::splat(4.0)), f32x4::from_array([4.0, 8.0, 10.0, -16.0])); - - assert_eq!(a.mul_add(b, a), (a * b) + a); - assert_eq!(b.mul_add(b, a), (b * b) + a); - assert_eq!(a.mul_add(b, b), (a * b) + b); - assert_eq!( - f32x4::splat(-3.2).mul_add(b, f32x4::splat(f32::NEG_INFINITY)), - f32x4::splat(f32::NEG_INFINITY) - ); - assert_eq!((a * a).sqrt(), a); - assert_eq!((b * b).sqrt(), b.abs()); - - assert_eq!(a.simd_eq(f32x4::splat(5.0) * b), Mask::from_array([false, true, false, false])); - assert_eq!(a.simd_ne(f32x4::splat(5.0) * b), Mask::from_array([true, false, true, true])); - assert_eq!(a.simd_le(f32x4::splat(5.0) * b), Mask::from_array([false, true, true, false])); - assert_eq!(a.simd_lt(f32x4::splat(5.0) * b), Mask::from_array([false, false, true, false])); - assert_eq!(a.simd_ge(f32x4::splat(5.0) * b), Mask::from_array([true, true, false, true])); - assert_eq!(a.simd_gt(f32x4::splat(5.0) * b), Mask::from_array([true, false, false, true])); - - assert_eq!(a.reduce_sum(), 40.0); - assert_eq!(b.reduce_sum(), 2.0); - assert_eq!(a.reduce_product(), 100.0 * 100.0); - assert_eq!(b.reduce_product(), -24.0); - assert_eq!(a.reduce_max(), 10.0); - assert_eq!(b.reduce_max(), 3.0); - assert_eq!(a.reduce_min(), 10.0); - assert_eq!(b.reduce_min(), -4.0); - - assert_eq!( - f32x2::from_array([0.0, f32::NAN]).simd_max(f32x2::from_array([f32::NAN, 0.0])), - f32x2::from_array([0.0, 0.0]) - ); - assert_eq!(f32x2::from_array([0.0, f32::NAN]).reduce_max(), 0.0); - assert_eq!(f32x2::from_array([f32::NAN, 0.0]).reduce_max(), 0.0); - assert_eq!( - f32x2::from_array([0.0, f32::NAN]).simd_min(f32x2::from_array([f32::NAN, 0.0])), - f32x2::from_array([0.0, 0.0]) - ); - assert_eq!(f32x2::from_array([0.0, f32::NAN]).reduce_min(), 0.0); - assert_eq!(f32x2::from_array([f32::NAN, 0.0]).reduce_min(), 0.0); -} - -fn simd_ops_f64() { - let a = f64x4::splat(10.0); - let b = f64x4::from_array([1.0, 2.0, 3.0, -4.0]); - assert_eq!(-b, f64x4::from_array([-1.0, -2.0, -3.0, 4.0])); - assert_eq!(a + b, f64x4::from_array([11.0, 12.0, 13.0, 6.0])); - assert_eq!(a - b, f64x4::from_array([9.0, 8.0, 7.0, 14.0])); - assert_eq!(a * b, f64x4::from_array([10.0, 20.0, 30.0, -40.0])); - assert_eq!(b / a, f64x4::from_array([0.1, 0.2, 0.3, -0.4])); - assert_eq!(a / f64x4::splat(2.0), f64x4::splat(5.0)); - assert_eq!(a % b, f64x4::from_array([0.0, 0.0, 1.0, 2.0])); - assert_eq!(b.abs(), f64x4::from_array([1.0, 2.0, 3.0, 4.0])); - assert_eq!(a.simd_max(b * f64x4::splat(4.0)), f64x4::from_array([10.0, 10.0, 12.0, 10.0])); - assert_eq!(a.simd_min(b * f64x4::splat(4.0)), f64x4::from_array([4.0, 8.0, 10.0, -16.0])); - - assert_eq!(a.mul_add(b, a), (a * b) + a); - assert_eq!(b.mul_add(b, a), (b * b) + a); - assert_eq!(a.mul_add(b, b), (a * b) + b); - assert_eq!( - f64x4::splat(-3.2).mul_add(b, f64x4::splat(f64::NEG_INFINITY)), - f64x4::splat(f64::NEG_INFINITY) - ); - assert_eq!((a * a).sqrt(), a); - assert_eq!((b * b).sqrt(), b.abs()); - - assert_eq!(a.simd_eq(f64x4::splat(5.0) * b), Mask::from_array([false, true, false, false])); - assert_eq!(a.simd_ne(f64x4::splat(5.0) * b), Mask::from_array([true, false, true, true])); - assert_eq!(a.simd_le(f64x4::splat(5.0) * b), Mask::from_array([false, true, true, false])); - assert_eq!(a.simd_lt(f64x4::splat(5.0) * b), Mask::from_array([false, false, true, false])); - assert_eq!(a.simd_ge(f64x4::splat(5.0) * b), Mask::from_array([true, true, false, true])); - assert_eq!(a.simd_gt(f64x4::splat(5.0) * b), Mask::from_array([true, false, false, true])); - - assert_eq!(a.reduce_sum(), 40.0); - assert_eq!(b.reduce_sum(), 2.0); - assert_eq!(a.reduce_product(), 100.0 * 100.0); - assert_eq!(b.reduce_product(), -24.0); - assert_eq!(a.reduce_max(), 10.0); - assert_eq!(b.reduce_max(), 3.0); - assert_eq!(a.reduce_min(), 10.0); - assert_eq!(b.reduce_min(), -4.0); - - assert_eq!( - f64x2::from_array([0.0, f64::NAN]).simd_max(f64x2::from_array([f64::NAN, 0.0])), - f64x2::from_array([0.0, 0.0]) - ); - assert_eq!(f64x2::from_array([0.0, f64::NAN]).reduce_max(), 0.0); - assert_eq!(f64x2::from_array([f64::NAN, 0.0]).reduce_max(), 0.0); - assert_eq!( - f64x2::from_array([0.0, f64::NAN]).simd_min(f64x2::from_array([f64::NAN, 0.0])), - f64x2::from_array([0.0, 0.0]) - ); - assert_eq!(f64x2::from_array([0.0, f64::NAN]).reduce_min(), 0.0); - assert_eq!(f64x2::from_array([f64::NAN, 0.0]).reduce_min(), 0.0); -} - -fn simd_ops_i32() { - let a = i32x4::splat(10); - let b = i32x4::from_array([1, 2, 3, -4]); - assert_eq!(-b, i32x4::from_array([-1, -2, -3, 4])); - assert_eq!(a + b, i32x4::from_array([11, 12, 13, 6])); - assert_eq!(a - b, i32x4::from_array([9, 8, 7, 14])); - assert_eq!(a * b, i32x4::from_array([10, 20, 30, -40])); - assert_eq!(a / b, i32x4::from_array([10, 5, 3, -2])); - assert_eq!(a / i32x4::splat(2), i32x4::splat(5)); - assert_eq!(i32x2::splat(i32::MIN) / i32x2::splat(-1), i32x2::splat(i32::MIN)); - assert_eq!(a % b, i32x4::from_array([0, 0, 1, 2])); - assert_eq!(i32x2::splat(i32::MIN) % i32x2::splat(-1), i32x2::splat(0)); - assert_eq!(b.abs(), i32x4::from_array([1, 2, 3, 4])); - assert_eq!(a.simd_max(b * i32x4::splat(4)), i32x4::from_array([10, 10, 12, 10])); - assert_eq!(a.simd_min(b * i32x4::splat(4)), i32x4::from_array([4, 8, 10, -16])); - - assert_eq!( - i8x4::from_array([i8::MAX, -23, 23, i8::MIN]).saturating_add(i8x4::from_array([ - 1, - i8::MIN, - i8::MAX, - 28 - ])), - i8x4::from_array([i8::MAX, i8::MIN, i8::MAX, -100]) - ); - assert_eq!( - i8x4::from_array([i8::MAX, -28, 27, 42]).saturating_sub(i8x4::from_array([ - 1, - i8::MAX, - i8::MAX, - -80 - ])), - i8x4::from_array([126, i8::MIN, -100, 122]) - ); - assert_eq!( - u8x4::from_array([u8::MAX, 0, 23, 42]).saturating_add(u8x4::from_array([ - 1, - 1, - u8::MAX, - 200 - ])), - u8x4::from_array([u8::MAX, 1, u8::MAX, 242]) - ); - assert_eq!( - u8x4::from_array([u8::MAX, 0, 23, 42]).saturating_sub(u8x4::from_array([ - 1, - 1, - u8::MAX, - 200 - ])), - u8x4::from_array([254, 0, 0, 0]) - ); - - assert_eq!(!b, i32x4::from_array([!1, !2, !3, !-4])); - assert_eq!(b << i32x4::splat(2), i32x4::from_array([4, 8, 12, -16])); - assert_eq!(b >> i32x4::splat(1), i32x4::from_array([0, 1, 1, -2])); - assert_eq!(b & i32x4::splat(2), i32x4::from_array([0, 2, 2, 0])); - assert_eq!(b | i32x4::splat(2), i32x4::from_array([3, 2, 3, -2])); - assert_eq!(b ^ i32x4::splat(2), i32x4::from_array([3, 0, 1, -2])); - - assert_eq!(a.simd_eq(i32x4::splat(5) * b), Mask::from_array([false, true, false, false])); - assert_eq!(a.simd_ne(i32x4::splat(5) * b), Mask::from_array([true, false, true, true])); - assert_eq!(a.simd_le(i32x4::splat(5) * b), Mask::from_array([false, true, true, false])); - assert_eq!(a.simd_lt(i32x4::splat(5) * b), Mask::from_array([false, false, true, false])); - assert_eq!(a.simd_ge(i32x4::splat(5) * b), Mask::from_array([true, true, false, true])); - assert_eq!(a.simd_gt(i32x4::splat(5) * b), Mask::from_array([true, false, false, true])); - - assert_eq!(a.reduce_sum(), 40); - assert_eq!(b.reduce_sum(), 2); - assert_eq!(a.reduce_product(), 100 * 100); - assert_eq!(b.reduce_product(), -24); - assert_eq!(a.reduce_max(), 10); - assert_eq!(b.reduce_max(), 3); - assert_eq!(a.reduce_min(), 10); - assert_eq!(b.reduce_min(), -4); - - assert_eq!(a.reduce_and(), 10); - assert_eq!(b.reduce_and(), 0); - assert_eq!(a.reduce_or(), 10); - assert_eq!(b.reduce_or(), -1); - assert_eq!(a.reduce_xor(), 0); - assert_eq!(b.reduce_xor(), -4); - - assert_eq!(b.leading_zeros(), u32x4::from_array([31, 30, 30, 0])); - assert_eq!(b.trailing_zeros(), u32x4::from_array([0, 1, 0, 2])); - assert_eq!(b.leading_ones(), u32x4::from_array([0, 0, 0, 30])); - assert_eq!(b.trailing_ones(), u32x4::from_array([1, 0, 2, 0])); - assert_eq!( - b.swap_bytes(), - i32x4::from_array([0x01000000, 0x02000000, 0x03000000, 0xfcffffffu32 as i32]) - ); - assert_eq!( - b.reverse_bits(), - i32x4::from_array([ - 0x80000000u32 as i32, - 0x40000000, - 0xc0000000u32 as i32, - 0x3fffffffu32 as i32 - ]) - ); -} - -fn simd_mask() { - use std::intrinsics::simd::*; - - let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0])); - assert_eq!(intmask, Mask::from_array([false, true, false, false])); - assert_eq!(intmask.to_array(), [false, true, false, false]); - - let values = [ - true, false, false, true, false, false, true, false, true, true, false, false, false, true, - false, true, - ]; - let mask = Mask::::from_array(values); - let bitmask = mask.to_bitmask(); - assert_eq!(bitmask, 0b1010001101001001); - assert_eq!(Mask::::from_bitmask(bitmask), mask); - - // Also directly call intrinsic, to test both kinds of return types. - unsafe { - let bitmask1: u16 = simd_bitmask(mask.to_int()); - let bitmask2: [u8; 2] = simd_bitmask(mask.to_int()); - if cfg!(target_endian = "little") { - assert_eq!(bitmask1, 0b1010001101001001); - assert_eq!(bitmask2, [0b01001001, 0b10100011]); - } else { - // All the bitstrings are reversed compared to above, but the array elements are in the - // same order. - assert_eq!(bitmask1, 0b1001001011000101); - assert_eq!(bitmask2, [0b10010010, 0b11000101]); - } - } - - // Mask less than 8 bits long, which is a special case (padding with 0s). - let values = [false, false, false, true]; - let mask = Mask::::from_array(values); - let bitmask = mask.to_bitmask(); - assert_eq!(bitmask, 0b1000); - assert_eq!(Mask::::from_bitmask(bitmask), mask); - unsafe { - let bitmask1: u8 = simd_bitmask(mask.to_int()); - let bitmask2: [u8; 1] = simd_bitmask(mask.to_int()); - if cfg!(target_endian = "little") { - assert_eq!(bitmask1, 0b1000); - assert_eq!(bitmask2, [0b1000]); - } else { - assert_eq!(bitmask1, 0b0001); - assert_eq!(bitmask2, [0b0001]); - } - } - - // This used to cause an ICE. It exercises simd_select_bitmask with an array as input. - let bitmask = u8x4::from_array([0b00001101, 0, 0, 0]); - assert_eq!( - mask32x4::from_bitmask_vector(bitmask), - mask32x4::from_array([true, false, true, true]), - ); - let bitmask = u8x8::from_array([0b01000101, 0, 0, 0, 0, 0, 0, 0]); - assert_eq!( - mask32x8::from_bitmask_vector(bitmask), - mask32x8::from_array([true, false, true, false, false, false, true, false]), - ); - let bitmask = - u8x16::from_array([0b01000101, 0b11110000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); - assert_eq!( - mask32x16::from_bitmask_vector(bitmask), - mask32x16::from_array([ - true, false, true, false, false, false, true, false, false, false, false, false, true, - true, true, true, - ]), - ); - - // Also directly call simd_select_bitmask, to test both kinds of argument types. - unsafe { - // These masks are exactly the results we got out above in the `simd_bitmask` tests. - let selected1 = simd_select_bitmask::( - if cfg!(target_endian = "little") { 0b1010001101001001 } else { 0b1001001011000101 }, - i32x16::splat(1), // yes - i32x16::splat(0), // no - ); - let selected2 = simd_select_bitmask::<[u8; 2], _>( - if cfg!(target_endian = "little") { - [0b01001001, 0b10100011] - } else { - [0b10010010, 0b11000101] - }, - i32x16::splat(1), // yes - i32x16::splat(0), // no - ); - assert_eq!(selected1, i32x16::from_array([1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1])); - assert_eq!(selected2, selected1); - // Also try masks less than a byte long. - let selected1 = simd_select_bitmask::( - if cfg!(target_endian = "little") { 0b1000 } else { 0b0001 }, - i32x4::splat(1), // yes - i32x4::splat(0), // no - ); - let selected2 = simd_select_bitmask::<[u8; 1], _>( - if cfg!(target_endian = "little") { [0b1000] } else { [0b0001] }, - i32x4::splat(1), // yes - i32x4::splat(0), // no - ); - assert_eq!(selected1, i32x4::from_array([0, 0, 0, 1])); - assert_eq!(selected2, selected1); - } -} - -fn simd_cast() { - // between integer types - assert_eq!(i32x4::from_array([1, 2, 3, -4]), i16x4::from_array([1, 2, 3, -4]).cast()); - assert_eq!(i16x4::from_array([1, 2, 3, -4]), i32x4::from_array([1, 2, 3, -4]).cast()); - assert_eq!(i32x4::from_array([1, -1, 3, 4]), u64x4::from_array([1, u64::MAX, 3, 4]).cast()); - - // float -> int - assert_eq!( - i8x4::from_array([127, -128, 127, -128]), - f32x4::from_array([127.99, -128.99, 999.0, -999.0]).cast() - ); - assert_eq!( - i32x4::from_array([0, 1, -1, 2147483520]), - f32x4::from_array([ - -0.0, - /*0x1.19999ap+0*/ f32::from_bits(0x3f8ccccd), - /*-0x1.19999ap+0*/ f32::from_bits(0xbf8ccccd), - 2147483520.0 - ]) - .cast() - ); - assert_eq!( - i32x8::from_array([i32::MAX, i32::MIN, i32::MAX, i32::MIN, i32::MAX, i32::MIN, 0, 0]), - f32x8::from_array([ - 2147483648.0f32, - -2147483904.0f32, - f32::MAX, - f32::MIN, - f32::INFINITY, - f32::NEG_INFINITY, - f32::NAN, - -f32::NAN, - ]) - .cast() - ); - - // int -> float - assert_eq!( - f32x4::from_array([ - -2147483648.0, - /*0x1.26580cp+30*/ f32::from_bits(0x4e932c06), - 16777220.0, - -16777220.0, - ]), - i32x4::from_array([-2147483647i32, 1234567890i32, 16777219i32, -16777219i32]).cast() - ); - - // float -> float - assert_eq!( - f32x4::from_array([f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY]), - f64x4::from_array([f64::MAX, f64::INFINITY, f64::MIN, f64::NEG_INFINITY]).cast() - ); - - // unchecked casts - unsafe { - assert_eq!( - i32x4::from_array([0, 1, -1, 2147483520]), - f32x4::from_array([ - -0.0, - /*0x1.19999ap+0*/ f32::from_bits(0x3f8ccccd), - /*-0x1.19999ap+0*/ f32::from_bits(0xbf8ccccd), - 2147483520.0 - ]) - .to_int_unchecked() - ); - assert_eq!( - u64x4::from_array([0, 10000000000000000, u64::MAX - 2047, 9223372036854775808]), - f64x4::from_array([ - -0.99999999999, - 1e16, - (u64::MAX - 1024) as f64, - 9223372036854775808.0 - ]) - .to_int_unchecked() - ); - } -} - -fn simd_swizzle() { - let a = f32x4::splat(10.0); - let b = f32x4::from_array([1.0, 2.0, 3.0, -4.0]); - - assert_eq!(simd_swizzle!(b, [3, 0, 0, 2]), f32x4::from_array([-4.0, 1.0, 1.0, 3.0])); - assert_eq!(simd_swizzle!(b, [1, 2]), f32x2::from_array([2.0, 3.0])); - assert_eq!(simd_swizzle!(b, a, [3, 4]), f32x2::from_array([-4.0, 10.0])); -} - -fn simd_gather_scatter() { - let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - let idxs = Simd::from_array([9, 3, 0, 17]); - let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. - assert_eq!(result, Simd::from_array([0, 13, 10, 0])); - - let idxs = Simd::from_array([9, 3, 0, 0]); - Simd::from_array([-27, 82, -41, 124]).scatter(&mut vec, idxs); - assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); - - // We call the intrinsics directly to experiment with dangling pointers and masks. - let val = 42u8; - let ptrs: Simd<*const u8, 4> = - Simd::from_array([ptr::null(), ptr::addr_of!(val), ptr::addr_of!(val), ptr::addr_of!(val)]); - let default = u8x4::splat(0); - let mask = i8x4::from_array([0, !0, 0, !0]); - let vals = unsafe { intrinsics::simd_gather(default, ptrs, mask) }; - assert_eq!(vals, u8x4::from_array([0, 42, 0, 42]),); - - let mut val1 = 0u8; - let mut val2 = 0u8; - let ptrs: Simd<*mut u8, 4> = Simd::from_array([ - ptr::null_mut(), - ptr::addr_of_mut!(val1), - ptr::addr_of_mut!(val1), - ptr::addr_of_mut!(val2), - ]); - let vals = u8x4::from_array([1, 2, 3, 4]); - unsafe { intrinsics::simd_scatter(vals, ptrs, mask) }; - assert_eq!(val1, 2); - assert_eq!(val2, 4); - - // Also check what happens when `scatter` has multiple overlapping pointers. - let mut val = 0u8; - let ptrs: Simd<*mut u8, 4> = Simd::from_array([ - ptr::addr_of_mut!(val), - ptr::addr_of_mut!(val), - ptr::addr_of_mut!(val), - ptr::addr_of_mut!(val), - ]); - let vals = u8x4::from_array([1, 2, 3, 4]); - unsafe { intrinsics::simd_scatter(vals, ptrs, mask) }; - assert_eq!(val, 4); -} - -fn simd_round() { - assert_eq!( - f32x4::from_array([0.9, 1.001, 2.0, -4.5]).ceil(), - f32x4::from_array([1.0, 2.0, 2.0, -4.0]) - ); - assert_eq!( - f32x4::from_array([0.9, 1.001, 2.0, -4.5]).floor(), - f32x4::from_array([0.0, 1.0, 2.0, -5.0]) - ); - assert_eq!( - f32x4::from_array([0.9, 1.001, 2.0, -4.5]).round(), - f32x4::from_array([1.0, 1.0, 2.0, -5.0]) - ); - assert_eq!( - f32x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(), - f32x4::from_array([0.0, 1.0, 2.0, -4.0]) - ); - - assert_eq!( - f64x4::from_array([0.9, 1.001, 2.0, -4.5]).ceil(), - f64x4::from_array([1.0, 2.0, 2.0, -4.0]) - ); - assert_eq!( - f64x4::from_array([0.9, 1.001, 2.0, -4.5]).floor(), - f64x4::from_array([0.0, 1.0, 2.0, -5.0]) - ); - assert_eq!( - f64x4::from_array([0.9, 1.001, 2.0, -4.5]).round(), - f64x4::from_array([1.0, 1.0, 2.0, -5.0]) - ); - assert_eq!( - f64x4::from_array([0.9, 1.001, 2.0, -4.5]).trunc(), - f64x4::from_array([0.0, 1.0, 2.0, -4.0]) - ); -} - -fn simd_intrinsics() { - use intrinsics::*; - - unsafe { - // Make sure simd_eq returns all-1 for `true` - let a = i32x4::splat(10); - let b = i32x4::from_array([1, 2, 10, 4]); - let c: i32x4 = simd_eq(a, b); - assert_eq!(c, i32x4::from_array([0, 0, -1, 0])); - - assert!(!simd_reduce_any(i32x4::splat(0))); - assert!(simd_reduce_any(i32x4::splat(-1))); - assert!(simd_reduce_any(i32x2::from_array([0, -1]))); - assert!(!simd_reduce_all(i32x4::splat(0))); - assert!(simd_reduce_all(i32x4::splat(-1))); - assert!(!simd_reduce_all(i32x2::from_array([0, -1]))); - - assert_eq!( - simd_select(i8x4::from_array([0, -1, -1, 0]), a, b), - i32x4::from_array([1, 10, 10, 4]) - ); - assert_eq!( - simd_select(i8x4::from_array([0, -1, -1, 0]), b, a), - i32x4::from_array([10, 2, 10, 10]) - ); - assert_eq!(simd_shuffle_generic::<_, i32x4, { &[3, 1, 0, 2] }>(a, b), a,); - assert_eq!(simd_shuffle::<_, _, i32x4>(a, b, const { [3u32, 1, 0, 2] }), a,); - assert_eq!( - simd_shuffle_generic::<_, i32x4, { &[7, 5, 4, 6] }>(a, b), - i32x4::from_array([4, 2, 1, 10]), - ); - assert_eq!( - simd_shuffle::<_, _, i32x4>(a, b, const { [7u32, 5, 4, 6] }), - i32x4::from_array([4, 2, 1, 10]), - ); - } -} - -fn simd_float_intrinsics() { - use intrinsics::*; - - // These are just smoke tests to ensure the intrinsics can be called. - unsafe { - let a = f32x4::splat(10.0); - simd_fsqrt(a); - simd_fsin(a); - simd_fcos(a); - simd_fexp(a); - simd_fexp2(a); - simd_flog(a); - simd_flog2(a); - simd_flog10(a); - } -} - -fn simd_masked_loadstore() { - // The buffer is deliberarely too short, so reading the last element would be UB. - let buf = [3i32; 3]; - let default = i32x4::splat(0); - let mask = i32x4::from_array([!0, !0, !0, 0]); - let vals = unsafe { intrinsics::simd_masked_load(mask, buf.as_ptr(), default) }; - assert_eq!(vals, i32x4::from_array([3, 3, 3, 0])); - // Also read in a way that the *first* element is OOB. - let mask2 = i32x4::from_array([0, !0, !0, !0]); - let vals = - unsafe { intrinsics::simd_masked_load(mask2, buf.as_ptr().wrapping_sub(1), default) }; - assert_eq!(vals, i32x4::from_array([0, 3, 3, 3])); - - // The buffer is deliberarely too short, so writing the last element would be UB. - let mut buf = [42i32; 3]; - let vals = i32x4::from_array([1, 2, 3, 4]); - unsafe { intrinsics::simd_masked_store(mask, buf.as_mut_ptr(), vals) }; - assert_eq!(buf, [1, 2, 3]); - // Also write in a way that the *first* element is OOB. - unsafe { intrinsics::simd_masked_store(mask2, buf.as_mut_ptr().wrapping_sub(1), vals) }; - assert_eq!(buf, [2, 3, 4]); -} - -fn main() { - simd_mask(); - simd_ops_f32(); - simd_ops_f64(); - simd_ops_i32(); - simd_cast(); - simd_swizzle(); - simd_gather_scatter(); - simd_round(); - simd_intrinsics(); - simd_float_intrinsics(); - simd_masked_loadstore(); -} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs new file mode 100644 index 00000000000..7363c753617 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-aes-vaes.rs @@ -0,0 +1,291 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=+aes,+vaes,+avx512f + +#![feature(avx512_target_feature, stdarch_x86_avx512)] + +use core::mem::transmute; +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +fn main() { + assert!(is_x86_feature_detected!("aes")); + assert!(is_x86_feature_detected!("vaes")); + assert!(is_x86_feature_detected!("avx512f")); + + unsafe { + test_aes(); + test_vaes(); + } +} + +// The constants in the tests below are just bit patterns. They should not +// be interpreted as integers; signedness does not make sense for them, but +// __m128i happens to be defined in terms of signed integers. +#[allow(overflowing_literals)] +#[target_feature(enable = "aes")] +unsafe fn test_aes() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/aes.rs + + #[target_feature(enable = "aes")] + unsafe fn test_mm_aesdec_si128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); + let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee); + let r = _mm_aesdec_si128(a, k); + assert_eq_m128i(r, e); + } + test_mm_aesdec_si128(); + + #[target_feature(enable = "aes")] + unsafe fn test_mm_aesdeclast_si128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); + let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493); + let r = _mm_aesdeclast_si128(a, k); + assert_eq_m128i(r, e); + } + test_mm_aesdeclast_si128(); + + #[target_feature(enable = "aes")] + unsafe fn test_mm_aesenc_si128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); + let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333); + let r = _mm_aesenc_si128(a, k); + assert_eq_m128i(r, e); + } + test_mm_aesenc_si128(); + + #[target_feature(enable = "aes")] + unsafe fn test_mm_aesenclast_si128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee); + let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8); + let r = _mm_aesenclast_si128(a, k); + assert_eq_m128i(r, e); + } + test_mm_aesenclast_si128(); + + #[target_feature(enable = "aes")] + unsafe fn test_mm_aesimc_si128() { + // Constants taken from https://msdn.microsoft.com/en-us/library/cc714195.aspx. + let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff); + let e = _mm_set_epi64x(0xc66c82284ee40aa0, 0x6633441122770055); + let r = _mm_aesimc_si128(a); + assert_eq_m128i(r, e); + } + test_mm_aesimc_si128(); +} + +// The constants in the tests below are just bit patterns. They should not +// be interpreted as integers; signedness does not make sense for them, but +// __m128i happens to be defined in terms of signed integers. +#[allow(overflowing_literals)] +#[target_feature(enable = "vaes,avx512f")] +unsafe fn test_vaes() { + #[target_feature(enable = "avx")] + unsafe fn get_a256() -> __m256i { + // Constants are random + _mm256_set_epi64x( + 0xb89f43a558d3cd51, + 0x57b3e81e369bd603, + 0xf177a1a626933fd6, + 0x50d8adbed1a2f9d7, + ) + } + #[target_feature(enable = "avx")] + unsafe fn get_k256() -> __m256i { + // Constants are random + _mm256_set_epi64x( + 0x503ff704588b5627, + 0xe23d882ed9c3c146, + 0x2785e5b670155b3c, + 0xa750718e183549ff, + ) + } + + #[target_feature(enable = "vaes")] + unsafe fn test_mm256_aesdec_epi128() { + let a = get_a256(); + let k = get_k256(); + let r = _mm256_aesdec_epi128(a, k); + + // Check results. + let a: [u128; 2] = transmute(a); + let k: [u128; 2] = transmute(k); + let r: [u128; 2] = transmute(r); + for i in 0..2 { + let e: u128 = transmute(_mm_aesdec_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm256_aesdec_epi128(); + + #[target_feature(enable = "vaes")] + unsafe fn test_mm256_aesdeclast_epi128() { + let a = get_a256(); + let k = get_k256(); + let r = _mm256_aesdeclast_epi128(a, k); + + // Check results. + let a: [u128; 2] = transmute(a); + let k: [u128; 2] = transmute(k); + let r: [u128; 2] = transmute(r); + for i in 0..2 { + let e: u128 = transmute(_mm_aesdeclast_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm256_aesdeclast_epi128(); + + #[target_feature(enable = "vaes")] + unsafe fn test_mm256_aesenc_epi128() { + let a = get_a256(); + let k = get_k256(); + let r = _mm256_aesenc_epi128(a, k); + + // Check results. + let a: [u128; 2] = transmute(a); + let k: [u128; 2] = transmute(k); + let r: [u128; 2] = transmute(r); + for i in 0..2 { + let e: u128 = transmute(_mm_aesenc_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm256_aesenc_epi128(); + + #[target_feature(enable = "vaes")] + unsafe fn test_mm256_aesenclast_epi128() { + let a = get_a256(); + let k = get_k256(); + let r = _mm256_aesenclast_epi128(a, k); + + // Check results. + let a: [u128; 2] = transmute(a); + let k: [u128; 2] = transmute(k); + let r: [u128; 2] = transmute(r); + for i in 0..2 { + let e: u128 = transmute(_mm_aesenclast_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm256_aesenclast_epi128(); + + #[target_feature(enable = "avx512f")] + unsafe fn get_a512() -> __m512i { + // Constants are random + _mm512_set_epi64( + 0xb89f43a558d3cd51, + 0x57b3e81e369bd603, + 0xf177a1a626933fd6, + 0x50d8adbed1a2f9d7, + 0xfbfee3116629db78, + 0x6aef4a91f2ad50f4, + 0x4258bb51ff1d476d, + 0x31da65761c8016cf, + ) + } + #[target_feature(enable = "avx512f")] + unsafe fn get_k512() -> __m512i { + // Constants are random + _mm512_set_epi64( + 0x503ff704588b5627, + 0xe23d882ed9c3c146, + 0x2785e5b670155b3c, + 0xa750718e183549ff, + 0xdfb408830a65d3d9, + 0x0de3d92adac81b0a, + 0xed2741fe12877cae, + 0x3251ddb5404e0974, + ) + } + + #[target_feature(enable = "vaes,avx512f")] + unsafe fn test_mm512_aesdec_epi128() { + let a = get_a512(); + let k = get_k512(); + let r = _mm512_aesdec_epi128(a, k); + + // Check results. + let a: [u128; 4] = transmute(a); + let k: [u128; 4] = transmute(k); + let r: [u128; 4] = transmute(r); + for i in 0..4 { + let e: u128 = transmute(_mm_aesdec_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm512_aesdec_epi128(); + + #[target_feature(enable = "vaes,avx512f")] + unsafe fn test_mm512_aesdeclast_epi128() { + let a = get_a512(); + let k = get_k512(); + let r = _mm512_aesdeclast_epi128(a, k); + + // Check results. + let a: [u128; 4] = transmute(a); + let k: [u128; 4] = transmute(k); + let r: [u128; 4] = transmute(r); + for i in 0..4 { + let e: u128 = transmute(_mm_aesdeclast_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm512_aesdeclast_epi128(); + + #[target_feature(enable = "vaes,avx512f")] + unsafe fn test_mm512_aesenc_epi128() { + let a = get_a512(); + let k = get_k512(); + let r = _mm512_aesenc_epi128(a, k); + + // Check results. + let a: [u128; 4] = transmute(a); + let k: [u128; 4] = transmute(k); + let r: [u128; 4] = transmute(r); + for i in 0..4 { + let e: u128 = transmute(_mm_aesenc_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm512_aesenc_epi128(); + + #[target_feature(enable = "vaes,avx512f")] + unsafe fn test_mm512_aesenclast_epi128() { + let a = get_a512(); + let k = get_k512(); + let r = _mm512_aesenclast_epi128(a, k); + + // Check results. + let a: [u128; 4] = transmute(a); + let k: [u128; 4] = transmute(k); + let r: [u128; 4] = transmute(r); + for i in 0..4 { + let e: u128 = transmute(_mm_aesenclast_si128(transmute(a[i]), transmute(k[i]))); + assert_eq!(r[i], e); + } + } + test_mm512_aesenclast_epi128(); +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs new file mode 100644 index 00000000000..7d43cc596ae --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx.rs @@ -0,0 +1,1431 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=+avx + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::mem::transmute; + +fn main() { + assert!(is_x86_feature_detected!("avx")); + + unsafe { + test_avx(); + } +} + +#[target_feature(enable = "avx")] +unsafe fn test_avx() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx.rs + + macro_rules! assert_approx_eq { + ($a:expr, $b:expr, $eps:expr) => {{ + let (a, b) = (&$a, &$b); + assert!( + (*a - *b).abs() < $eps, + "assertion failed: `(left !== right)` \ + (left: `{:?}`, right: `{:?}`, expect diff: `{:?}`, real diff: `{:?}`)", + *a, + *b, + $eps, + (*a - *b).abs() + ); + }}; + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_max_pd() { + let a = _mm256_setr_pd(1., 4., 5., 8.); + let b = _mm256_setr_pd(2., 3., 6., 7.); + let r = _mm256_max_pd(a, b); + let e = _mm256_setr_pd(2., 4., 6., 8.); + assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_max_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_max_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); + } + test_mm256_max_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_max_ps() { + let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_max_ps(a, b); + let e = _mm256_setr_ps(2., 4., 6., 8., 10., 12., 14., 16.); + assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_max_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_max_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_max_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); + } + test_mm256_max_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_min_pd() { + let a = _mm256_setr_pd(1., 4., 5., 8.); + let b = _mm256_setr_pd(2., 3., 6., 7.); + let r = _mm256_min_pd(a, b); + let e = _mm256_setr_pd(1., 3., 5., 7.); + assert_eq_m256d(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(-0.0)); + let x = _mm256_min_pd(_mm256_set1_pd(-0.0), _mm256_set1_pd(0.0)); + let wu: [u64; 4] = transmute(w); + let xu: [u64; 4] = transmute(x); + assert_eq!(wu, [0x8000_0000_0000_0000u64; 4]); + assert_eq!(xu, [0u64; 4]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_pd(_mm256_set1_pd(f64::NAN), _mm256_set1_pd(0.0)); + let z = _mm256_min_pd(_mm256_set1_pd(0.0), _mm256_set1_pd(f64::NAN)); + let yf: [f64; 4] = transmute(y); + let zf: [f64; 4] = transmute(z); + assert_eq!(yf, [0.0; 4]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); + } + test_mm256_min_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_min_ps() { + let a = _mm256_setr_ps(1., 4., 5., 8., 9., 12., 13., 16.); + let b = _mm256_setr_ps(2., 3., 6., 7., 10., 11., 14., 15.); + let r = _mm256_min_ps(a, b); + let e = _mm256_setr_ps(1., 3., 5., 7., 9., 11., 13., 15.); + assert_eq_m256(r, e); + // > If the values being compared are both 0.0s (of either sign), the + // > value in the second operand (source operand) is returned. + let w = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(-0.0)); + let x = _mm256_min_ps(_mm256_set1_ps(-0.0), _mm256_set1_ps(0.0)); + let wu: [u32; 8] = transmute(w); + let xu: [u32; 8] = transmute(x); + assert_eq!(wu, [0x8000_0000u32; 8]); + assert_eq!(xu, [0u32; 8]); + // > If only one value is a NaN (SNaN or QNaN) for this instruction, the + // > second operand (source operand), either a NaN or a valid + // > floating-point value, is written to the result. + let y = _mm256_min_ps(_mm256_set1_ps(f32::NAN), _mm256_set1_ps(0.0)); + let z = _mm256_min_ps(_mm256_set1_ps(0.0), _mm256_set1_ps(f32::NAN)); + let yf: [f32; 8] = transmute(y); + let zf: [f32; 8] = transmute(z); + assert_eq!(yf, [0.0; 8]); + assert!(zf.iter().all(|f| f.is_nan()), "{:?}", zf); + } + test_mm256_min_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_nearest_f32() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f32, res: f32) { + let a = _mm256_set1_ps(x); + let e = _mm256_set1_ps(res); + let r = _mm256_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m256(r, e); + // Assume round-to-nearest by default + let r = _mm256_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m256(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); + let e = _mm256_setr_ps(2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0); + let r = _mm256_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m256(r, e); + // Assume round-to-nearest by default + let r = _mm256_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m256(r, e); + } + test_round_nearest_f32(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_floor_f32() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f32, res: f32) { + let a = _mm256_set1_ps(x); + let e = _mm256_set1_ps(res); + let r = _mm256_floor_ps(a); + assert_eq_m256(r, e); + let r = _mm256_round_ps::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m256(r, e); + } + + // Test rounding direction + test(-2.5, -3.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -2.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); + let e = _mm256_setr_ps(1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0); + let r = _mm256_floor_ps(a); + assert_eq_m256(r, e); + let r = _mm256_round_ps::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m256(r, e); + } + test_round_floor_f32(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_ceil_f32() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f32, res: f32) { + let a = _mm256_set1_ps(x); + let e = _mm256_set1_ps(res); + let r = _mm256_ceil_ps(a); + assert_eq_m256(r, e); + let r = _mm256_round_ps::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m256(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 2.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 3.0); + + // Test that each element is rounded + let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); + let e = _mm256_setr_ps(2.0, 4.0, 6.0, 8.0, 10.0, 12.0, 14.0, 16.0); + let r = _mm256_ceil_ps(a); + assert_eq_m256(r, e); + let r = _mm256_round_ps::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m256(r, e); + } + test_round_ceil_f32(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_trunc_f32() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f32, res: f32) { + let a = _mm256_set1_ps(x); + let e = _mm256_set1_ps(res); + let r = _mm256_round_ps::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m256(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_ps(1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5, 15.5); + let e = _mm256_setr_ps(1.0, 3.0, 5.0, 7.0, 9.0, 11.0, 13.0, 15.0); + let r = _mm256_round_ps::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m256(r, e); + } + test_round_trunc_f32(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_nearest_f64() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f64, res: f64) { + let a = _mm256_set1_pd(x); + let e = _mm256_set1_pd(res); + let r = _mm256_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m256d(r, e); + // Assume round-to-nearest by default + let r = _mm256_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m256d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); + let e = _mm256_setr_pd(2.0, 4.0, 6.0, 8.0); + let r = _mm256_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m256d(r, e); + // Assume round-to-nearest by default + let r = _mm256_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m256d(r, e); + } + test_round_nearest_f64(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_floor_f64() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f64, res: f64) { + let a = _mm256_set1_pd(x); + let e = _mm256_set1_pd(res); + let r = _mm256_floor_pd(a); + assert_eq_m256d(r, e); + let r = _mm256_round_pd::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m256d(r, e); + } + + // Test rounding direction + test(-2.5, -3.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -2.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); + let e = _mm256_setr_pd(1.0, 3.0, 5.0, 7.0); + let r = _mm256_floor_pd(a); + assert_eq_m256d(r, e); + let r = _mm256_round_pd::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m256d(r, e); + } + test_round_floor_f64(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_ceil_f64() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f64, res: f64) { + let a = _mm256_set1_pd(x); + let e = _mm256_set1_pd(res); + let r = _mm256_ceil_pd(a); + assert_eq_m256d(r, e); + let r = _mm256_round_pd::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m256d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 2.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 3.0); + + // Test that each element is rounded + let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); + let e = _mm256_setr_pd(2.0, 4.0, 6.0, 8.0); + let r = _mm256_ceil_pd(a); + assert_eq_m256d(r, e); + let r = _mm256_round_pd::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m256d(r, e); + } + test_round_ceil_f64(); + + #[target_feature(enable = "avx")] + unsafe fn test_round_trunc_f64() { + #[target_feature(enable = "avx")] + unsafe fn test(x: f64, res: f64) { + let a = _mm256_set1_pd(x); + let e = _mm256_set1_pd(res); + let r = _mm256_round_pd::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m256d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm256_setr_pd(1.5, 3.5, 5.5, 7.5); + let e = _mm256_setr_pd(1.0, 3.0, 5.0, 7.0); + let r = _mm256_round_pd::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m256d(r, e); + } + test_round_trunc_f64(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_sqrt_ps() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let r = _mm256_sqrt_ps(a); + let e = _mm256_setr_ps(2., 3., 4., 5., 2., 3., 4., 5.); + assert_eq_m256(r, e); + } + test_mm256_sqrt_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_rcp_ps() { + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_rcp_ps(a); + #[rustfmt::skip] + let e = _mm256_setr_ps( + 0.99975586, 0.49987793, 0.33325195, 0.24993896, + 0.19995117, 0.16662598, 0.14282227, 0.12496948, + ); + let rel_err = 0.00048828125; + + let r: [f32; 8] = transmute(r); + let e: [f32; 8] = transmute(e); + for i in 0..8 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm256_rcp_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_rsqrt_ps() { + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let r = _mm256_rsqrt_ps(a); + #[rustfmt::skip] + let e = _mm256_setr_ps( + 0.99975586, 0.7069092, 0.5772705, 0.49987793, + 0.44714355, 0.40820313, 0.3779297, 0.3534546, + ); + let rel_err = 0.00048828125; + + let r: [f32; 8] = transmute(r); + let e: [f32; 8] = transmute(e); + for i in 0..8 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm256_rsqrt_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_dp_ps() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); + let r = _mm256_dp_ps::<0xFF>(a, b); + let e = _mm256_setr_ps(200., 200., 200., 200., 2387., 2387., 2387., 2387.); + assert_eq_m256(r, e); + } + test_mm256_dp_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_hadd_pd() { + let a = _mm256_setr_pd(4., 9., 16., 25.); + let b = _mm256_setr_pd(4., 3., 2., 5.); + let r = _mm256_hadd_pd(a, b); + let e = _mm256_setr_pd(13., 7., 41., 7.); + assert_eq_m256d(r, e); + + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_hadd_pd(a, b); + let e = _mm256_setr_pd(3., 11., 7., 15.); + assert_eq_m256d(r, e); + } + test_mm256_hadd_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_hadd_ps() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); + let r = _mm256_hadd_ps(a, b); + let e = _mm256_setr_ps(13., 41., 7., 7., 13., 41., 17., 114.); + assert_eq_m256(r, e); + + let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); + let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); + let r = _mm256_hadd_ps(a, b); + let e = _mm256_setr_ps(3., 7., 11., 15., 3., 7., 11., 15.); + assert_eq_m256(r, e); + } + test_mm256_hadd_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_hsub_pd() { + let a = _mm256_setr_pd(4., 9., 16., 25.); + let b = _mm256_setr_pd(4., 3., 2., 5.); + let r = _mm256_hsub_pd(a, b); + let e = _mm256_setr_pd(-5., 1., -9., -3.); + assert_eq_m256d(r, e); + + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_hsub_pd(a, b); + let e = _mm256_setr_pd(-1., -1., -1., -1.); + assert_eq_m256d(r, e); + } + test_mm256_hsub_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_hsub_ps() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let b = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); + let r = _mm256_hsub_ps(a, b); + let e = _mm256_setr_ps(-5., -9., 1., -3., -5., -9., -1., 14.); + assert_eq_m256(r, e); + + let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); + let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); + let r = _mm256_hsub_ps(a, b); + let e = _mm256_setr_ps(-1., -1., -1., -1., -1., -1., -1., -1.); + assert_eq_m256(r, e); + } + test_mm256_hsub_ps(); + + fn expected_cmp(imm: i32, lhs: F, rhs: F, if_t: F, if_f: F) -> F { + let res = match imm { + _CMP_EQ_OQ => lhs == rhs, + _CMP_LT_OS => lhs < rhs, + _CMP_LE_OS => lhs <= rhs, + _CMP_UNORD_Q => lhs.partial_cmp(&rhs).is_none(), + _CMP_NEQ_UQ => lhs != rhs, + _CMP_NLT_UQ => !(lhs < rhs), + _CMP_NLE_UQ => !(lhs <= rhs), + _CMP_ORD_Q => lhs.partial_cmp(&rhs).is_some(), + _CMP_EQ_UQ => lhs == rhs || lhs.partial_cmp(&rhs).is_none(), + _CMP_NGE_US => !(lhs >= rhs), + _CMP_NGT_US => !(lhs > rhs), + _CMP_FALSE_OQ => false, + _CMP_NEQ_OQ => lhs != rhs && lhs.partial_cmp(&rhs).is_some(), + _CMP_GE_OS => lhs >= rhs, + _CMP_GT_OS => lhs > rhs, + _CMP_TRUE_US => true, + _ => unreachable!(), + }; + if res { if_t } else { if_f } + } + fn expected_cmp_f32(imm: i32, lhs: f32, rhs: f32) -> f32 { + expected_cmp(imm, lhs, rhs, f32::from_bits(u32::MAX), 0.0) + } + fn expected_cmp_f64(imm: i32, lhs: f64, rhs: f64) -> f64 { + expected_cmp(imm, lhs, rhs, f64::from_bits(u64::MAX), 0.0) + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm_cmp_ss() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f32::NAN, 0.0), + (0.0, f32::NAN), + (f32::NAN, f32::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm_setr_ps(lhs, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(rhs, 5.0, 6.0, 7.0); + let r: [u32; 4] = transmute(_mm_cmp_ss::(a, b)); + let e: [u32; 4] = + transmute(_mm_setr_ps(expected_cmp_f32(IMM, lhs, rhs), 2.0, 3.0, 4.0)); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm_cmp_ps() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f32::NAN, 0.0), + (0.0, f32::NAN), + (f32::NAN, f32::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm_set1_ps(lhs); + let b = _mm_set1_ps(rhs); + let r: [u32; 4] = transmute(_mm_cmp_ps::(a, b)); + let e: [u32; 4] = transmute(_mm_set1_ps(expected_cmp_f32(IMM, lhs, rhs))); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm_cmp_sd() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f64::NAN, 0.0), + (0.0, f64::NAN), + (f64::NAN, f64::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm_setr_pd(lhs, 2.0); + let b = _mm_setr_pd(rhs, 3.0); + let r: [u64; 2] = transmute(_mm_cmp_sd::(a, b)); + let e: [u64; 2] = transmute(_mm_setr_pd(expected_cmp_f64(IMM, lhs, rhs), 2.0)); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm_cmp_pd() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f64::NAN, 0.0), + (0.0, f64::NAN), + (f64::NAN, f64::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm_set1_pd(lhs); + let b = _mm_set1_pd(rhs); + let r: [u64; 2] = transmute(_mm_cmp_pd::(a, b)); + let e: [u64; 2] = transmute(_mm_set1_pd(expected_cmp_f64(IMM, lhs, rhs))); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cmp_ps() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f32::NAN, 0.0), + (0.0, f32::NAN), + (f32::NAN, f32::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm256_set1_ps(lhs); + let b = _mm256_set1_ps(rhs); + let r: [u32; 8] = transmute(_mm256_cmp_ps::(a, b)); + let e: [u32; 8] = transmute(_mm256_set1_ps(expected_cmp_f32(IMM, lhs, rhs))); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cmp_pd() { + let values = [ + (1.0, 1.0), + (0.0, 1.0), + (1.0, 0.0), + (f64::NAN, 0.0), + (0.0, f64::NAN), + (f64::NAN, f64::NAN), + ]; + + for (lhs, rhs) in values { + let a = _mm256_set1_pd(lhs); + let b = _mm256_set1_pd(rhs); + let r: [u64; 4] = transmute(_mm256_cmp_pd::(a, b)); + let e: [u64; 4] = transmute(_mm256_set1_pd(expected_cmp_f64(IMM, lhs, rhs))); + assert_eq!(r, e); + } + } + + #[target_feature(enable = "avx")] + unsafe fn test_cmp() { + test_mm_cmp_ss::(); + test_mm_cmp_ps::(); + test_mm_cmp_sd::(); + test_mm_cmp_pd::(); + test_mm256_cmp_ps::(); + test_mm256_cmp_pd::(); + } + + test_cmp::<_CMP_EQ_OQ>(); + test_cmp::<_CMP_LT_OS>(); + test_cmp::<_CMP_LE_OS>(); + test_cmp::<_CMP_UNORD_Q>(); + test_cmp::<_CMP_NEQ_UQ>(); + test_cmp::<_CMP_NLT_UQ>(); + test_cmp::<_CMP_NLE_UQ>(); + test_cmp::<_CMP_ORD_Q>(); + test_cmp::<_CMP_EQ_UQ>(); + test_cmp::<_CMP_NGE_US>(); + test_cmp::<_CMP_NGT_US>(); + test_cmp::<_CMP_FALSE_OQ>(); + test_cmp::<_CMP_NEQ_OQ>(); + test_cmp::<_CMP_GE_OS>(); + test_cmp::<_CMP_GT_OS>(); + test_cmp::<_CMP_TRUE_US>(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cvtps_epi32() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let r = _mm256_cvtps_epi32(a); + let e = _mm256_setr_epi32(4, 9, 16, 25, 4, 9, 16, 25); + assert_eq_m256i(r, e); + + let a = _mm256_setr_ps( + f32::NEG_INFINITY, + f32::INFINITY, + f32::MIN, + f32::MAX, + f32::NAN, + f32::NAN, + f32::NAN, + f32::NAN, + ); + let r = _mm256_cvtps_epi32(a); + assert_eq_m256i(r, _mm256_set1_epi32(i32::MIN)); + } + test_mm256_cvtps_epi32(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cvttps_epi32() { + let a = _mm256_setr_ps(4., 9., 16., 25., 4., 9., 16., 25.); + let r = _mm256_cvttps_epi32(a); + let e = _mm256_setr_epi32(4, 9, 16, 25, 4, 9, 16, 25); + assert_eq_m256i(r, e); + + let a = _mm256_setr_ps( + f32::NEG_INFINITY, + f32::INFINITY, + f32::MIN, + f32::MAX, + f32::NAN, + f32::NAN, + f32::NAN, + f32::NAN, + ); + let r = _mm256_cvttps_epi32(a); + assert_eq_m256i(r, _mm256_set1_epi32(i32::MIN)); + } + test_mm256_cvttps_epi32(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cvtpd_epi32() { + let a = _mm256_setr_pd(4., 9., 16., 25.); + let r = _mm256_cvtpd_epi32(a); + let e = _mm_setr_epi32(4, 9, 16, 25); + assert_eq_m128i(r, e); + + let a = _mm256_setr_pd(f64::NEG_INFINITY, f64::INFINITY, f64::MIN, f64::MAX); + let r = _mm256_cvtpd_epi32(a); + assert_eq_m128i(r, _mm_set1_epi32(i32::MIN)); + } + test_mm256_cvtpd_epi32(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_cvttpd_epi32() { + let a = _mm256_setr_pd(4., 9., 16., 25.); + let r = _mm256_cvttpd_epi32(a); + let e = _mm_setr_epi32(4, 9, 16, 25); + assert_eq_m128i(r, e); + + let a = _mm256_setr_pd(f64::NEG_INFINITY, f64::INFINITY, f64::MIN, f64::MAX); + let r = _mm256_cvttpd_epi32(a); + assert_eq_m128i(r, _mm_set1_epi32(i32::MIN)); + } + test_mm256_cvttpd_epi32(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_permutevar_ps() { + let a = _mm_setr_ps(4., 3., 2., 5.); + let b = _mm_setr_epi32(1, 2, 3, 4); + let r = _mm_permutevar_ps(a, b); + let e = _mm_setr_ps(3., 2., 5., 4.); + assert_eq_m128(r, e); + } + test_mm_permutevar_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_permutevar_ps() { + let a = _mm256_setr_ps(4., 3., 2., 5., 8., 9., 64., 50.); + let b = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let r = _mm256_permutevar_ps(a, b); + let e = _mm256_setr_ps(3., 2., 5., 4., 9., 64., 50., 8.); + assert_eq_m256(r, e); + } + test_mm256_permutevar_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_permutevar_pd() { + let a = _mm_setr_pd(4., 3.); + let b = _mm_setr_epi64x(3, 0); + let r = _mm_permutevar_pd(a, b); + let e = _mm_setr_pd(3., 4.); + assert_eq_m128d(r, e); + } + test_mm_permutevar_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_permutevar_pd() { + let a = _mm256_setr_pd(4., 3., 2., 5.); + let b = _mm256_setr_epi64x(1, 2, 3, 4); + let r = _mm256_permutevar_pd(a, b); + let e = _mm256_setr_pd(4., 3., 5., 2.); + assert_eq_m256d(r, e); + } + test_mm256_permutevar_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_permute2f128_ps() { + let a = _mm256_setr_ps(1., 2., 3., 4., 1., 2., 3., 4.); + let b = _mm256_setr_ps(5., 6., 7., 8., 5., 6., 7., 8.); + let r = _mm256_permute2f128_ps::<0x13>(a, b); + let e = _mm256_setr_ps(5., 6., 7., 8., 1., 2., 3., 4.); + assert_eq_m256(r, e); + + let r = _mm256_permute2f128_ps::<0x44>(a, b); + let e = _mm256_setr_ps(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); + assert_eq_m256(r, e); + } + test_mm256_permute2f128_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_permute2f128_pd() { + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_permute2f128_pd::<0x31>(a, b); + let e = _mm256_setr_pd(3., 4., 7., 8.); + assert_eq_m256d(r, e); + + let r = _mm256_permute2f128_pd::<0x44>(a, b); + let e = _mm256_setr_pd(0.0, 0.0, 0.0, 0.0); + assert_eq_m256d(r, e); + } + test_mm256_permute2f128_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_permute2f128_si256() { + let a = _mm256_setr_epi32(1, 2, 3, 4, 1, 2, 3, 4); + let b = _mm256_setr_epi32(5, 6, 7, 8, 5, 6, 7, 8); + let r = _mm256_permute2f128_si256::<0x20>(a, b); + let e = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + assert_eq_m256i(r, e); + + let r = _mm256_permute2f128_si256::<0x44>(a, b); + let e = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + } + test_mm256_permute2f128_si256(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_maskload_ps() { + let a = &[1.0f32, 2., 3., 4.]; + let mask = _mm_setr_epi32(0, !0, 0, !0); + let r = _mm_maskload_ps(a.as_ptr(), mask); + let e = _mm_setr_ps(0., 2., 0., 4.); + assert_eq_m128(r, e); + + // Unaligned pointer + let a = Unaligned::new([1.0f32, 2., 3., 4.]); + let mask = _mm_setr_epi32(0, !0, 0, !0); + let r = _mm_maskload_ps(a.as_ptr().cast(), mask); + let e = _mm_setr_ps(0., 2., 0., 4.); + assert_eq_m128(r, e); + + // Only loading first element, so slice can be short. + let a = &[2.0f32]; + let mask = _mm_setr_epi32(!0, 0, 0, 0); + let r = _mm_maskload_ps(a.as_ptr(), mask); + let e = _mm_setr_ps(2.0, 0.0, 0.0, 0.0); + assert_eq_m128(r, e); + + // Only loading last element, so slice can be short. + let a = &[2.0f32]; + let mask = _mm_setr_epi32(0, 0, 0, !0); + let r = _mm_maskload_ps(a.as_ptr().wrapping_sub(3), mask); + let e = _mm_setr_ps(0.0, 0.0, 0.0, 2.0); + assert_eq_m128(r, e); + } + test_mm_maskload_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_maskload_pd() { + let a = &[1.0f64, 2.]; + let mask = _mm_setr_epi64x(0, !0); + let r = _mm_maskload_pd(a.as_ptr(), mask); + let e = _mm_setr_pd(0., 2.); + assert_eq_m128d(r, e); + + // Unaligned pointer + let a = Unaligned::new([1.0f64, 2.]); + let mask = _mm_setr_epi64x(0, !0); + let r = _mm_maskload_pd(a.as_ptr().cast(), mask); + let e = _mm_setr_pd(0., 2.); + assert_eq_m128d(r, e); + + // Only loading first element, so slice can be short. + let a = &[2.0f64]; + let mask = _mm_setr_epi64x(!0, 0); + let r = _mm_maskload_pd(a.as_ptr(), mask); + let e = _mm_setr_pd(2.0, 0.0); + assert_eq_m128d(r, e); + + // Only loading last element, so slice can be short. + let a = &[2.0f64]; + let mask = _mm_setr_epi64x(0, !0); + let r = _mm_maskload_pd(a.as_ptr().wrapping_sub(1), mask); + let e = _mm_setr_pd(0.0, 2.0); + assert_eq_m128d(r, e); + } + test_mm_maskload_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_maskload_ps() { + let a = &[1.0f32, 2., 3., 4., 5., 6., 7., 8.]; + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let r = _mm256_maskload_ps(a.as_ptr(), mask); + let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); + assert_eq_m256(r, e); + + // Unaligned pointer + let a = Unaligned::new([1.0f32, 2., 3., 4., 5., 6., 7., 8.]); + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let r = _mm256_maskload_ps(a.as_ptr().cast(), mask); + let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); + assert_eq_m256(r, e); + + // Only loading first element, so slice can be short. + let a = &[2.0f32]; + let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_maskload_ps(a.as_ptr(), mask); + let e = _mm256_setr_ps(2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); + assert_eq_m256(r, e); + + // Only loading last element, so slice can be short. + let a = &[2.0f32]; + let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); + let r = _mm256_maskload_ps(a.as_ptr().wrapping_sub(7), mask); + let e = _mm256_setr_ps(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0); + assert_eq_m256(r, e); + } + test_mm256_maskload_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_maskload_pd() { + let a = &[1.0f64, 2., 3., 4.]; + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let r = _mm256_maskload_pd(a.as_ptr(), mask); + let e = _mm256_setr_pd(0., 2., 0., 4.); + assert_eq_m256d(r, e); + + // Unaligned pointer + let a = Unaligned::new([1.0f64, 2., 3., 4.]); + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let r = _mm256_maskload_pd(a.as_ptr().cast(), mask); + let e = _mm256_setr_pd(0., 2., 0., 4.); + assert_eq_m256d(r, e); + + // Only loading first element, so slice can be short. + let a = &[2.0f64]; + let mask = _mm256_setr_epi64x(!0, 0, 0, 0); + let r = _mm256_maskload_pd(a.as_ptr(), mask); + let e = _mm256_setr_pd(2.0, 0.0, 0.0, 0.0); + assert_eq_m256d(r, e); + + // Only loading last element, so slice can be short. + let a = &[2.0f64]; + let mask = _mm256_setr_epi64x(0, 0, 0, !0); + let r = _mm256_maskload_pd(a.as_ptr().wrapping_sub(3), mask); + let e = _mm256_setr_pd(0.0, 0.0, 0.0, 2.0); + assert_eq_m256d(r, e); + } + test_mm256_maskload_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_maskstore_ps() { + let mut r = _mm_set1_ps(0.); + let mask = _mm_setr_epi32(0, !0, 0, !0); + let a = _mm_setr_ps(1., 2., 3., 4.); + _mm_maskstore_ps(&mut r as *mut _ as *mut f32, mask, a); + let e = _mm_setr_ps(0., 2., 0., 4.); + assert_eq_m128(r, e); + + // Unaligned pointer + let mut r = Unaligned::new([0.0f32; 4]); + let mask = _mm_setr_epi32(0, !0, 0, !0); + let a = _mm_setr_ps(1., 2., 3., 4.); + _mm_maskstore_ps(r.as_mut_ptr().cast(), mask, a); + let e = [0., 2., 0., 4.]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0.0f32]; + let mask = _mm_setr_epi32(!0, 0, 0, 0); + let a = _mm_setr_ps(1., 2., 3., 4.); + _mm_maskstore_ps(r.as_mut_ptr(), mask, a); + let e = [1.0f32]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0.0f32]; + let mask = _mm_setr_epi32(0, 0, 0, !0); + let a = _mm_setr_ps(1., 2., 3., 4.); + _mm_maskstore_ps(r.as_mut_ptr().wrapping_sub(3), mask, a); + let e = [4.0f32]; + assert_eq!(r, e); + } + test_mm_maskstore_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_maskstore_pd() { + let mut r = _mm_set1_pd(0.); + let mask = _mm_setr_epi64x(0, !0); + let a = _mm_setr_pd(1., 2.); + _mm_maskstore_pd(&mut r as *mut _ as *mut f64, mask, a); + let e = _mm_setr_pd(0., 2.); + assert_eq_m128d(r, e); + + // Unaligned pointer + let mut r = Unaligned::new([0.0f64; 2]); + let mask = _mm_setr_epi64x(0, !0); + let a = _mm_setr_pd(1., 2.); + _mm_maskstore_pd(r.as_mut_ptr().cast(), mask, a); + let e = [0., 2.]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0.0f64]; + let mask = _mm_setr_epi64x(!0, 0); + let a = _mm_setr_pd(1., 2.); + _mm_maskstore_pd(r.as_mut_ptr(), mask, a); + let e = [1.0f64]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0.0f64]; + let mask = _mm_setr_epi64x(0, !0); + let a = _mm_setr_pd(1., 2.); + _mm_maskstore_pd(r.as_mut_ptr().wrapping_sub(1), mask, a); + let e = [2.0f64]; + assert_eq!(r, e); + } + test_mm_maskstore_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_maskstore_ps() { + let mut r = _mm256_set1_ps(0.); + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + _mm256_maskstore_ps(&mut r as *mut _ as *mut f32, mask, a); + let e = _mm256_setr_ps(0., 2., 0., 4., 0., 6., 0., 8.); + assert_eq_m256(r, e); + + // Unaligned pointer + let mut r = Unaligned::new([0.0f32; 8]); + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + _mm256_maskstore_ps(r.as_mut_ptr().cast(), mask, a); + let e = [0., 2., 0., 4., 0., 6., 0., 8.]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0.0f32]; + let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + _mm256_maskstore_ps(r.as_mut_ptr(), mask, a); + let e = [1.0f32]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0.0f32]; + let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + _mm256_maskstore_ps(r.as_mut_ptr().wrapping_sub(7), mask, a); + let e = [8.0f32]; + assert_eq!(r, e); + } + test_mm256_maskstore_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_maskstore_pd() { + let mut r = _mm256_set1_pd(0.); + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let a = _mm256_setr_pd(1., 2., 3., 4.); + _mm256_maskstore_pd(&mut r as *mut _ as *mut f64, mask, a); + let e = _mm256_setr_pd(0., 2., 0., 4.); + assert_eq_m256d(r, e); + + // Unaligned pointer + let mut r = Unaligned::new([0.0f64; 4]); + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let a = _mm256_setr_pd(1., 2., 3., 4.); + _mm256_maskstore_pd(r.as_mut_ptr().cast(), mask, a); + let e = [0., 2., 0., 4.]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0.0f64]; + let mask = _mm256_setr_epi64x(!0, 0, 0, 0); + let a = _mm256_setr_pd(1., 2., 3., 4.); + _mm256_maskstore_pd(r.as_mut_ptr(), mask, a); + let e = [1.0f64]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0.0f64]; + let mask = _mm256_setr_epi64x(0, 0, 0, !0); + let a = _mm256_setr_pd(1., 2., 3., 4.); + _mm256_maskstore_pd(r.as_mut_ptr().wrapping_sub(3), mask, a); + let e = [4.0f64]; + assert_eq!(r, e); + } + test_mm256_maskstore_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_lddqu_si256() { + #[rustfmt::skip] + let a = _mm256_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ); + let p = &a as *const _; + let r = _mm256_lddqu_si256(p); + #[rustfmt::skip] + let e = _mm256_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ); + assert_eq_m256i(r, e); + } + test_mm256_lddqu_si256(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testz_si256() { + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let b = _mm256_setr_epi64x(5, 6, 7, 8); + let r = _mm256_testz_si256(a, b); + assert_eq!(r, 0); + let b = _mm256_set1_epi64x(0); + let r = _mm256_testz_si256(a, b); + assert_eq!(r, 1); + } + test_mm256_testz_si256(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testc_si256() { + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let b = _mm256_setr_epi64x(5, 6, 7, 8); + let r = _mm256_testc_si256(a, b); + assert_eq!(r, 0); + let b = _mm256_set1_epi64x(0); + let r = _mm256_testc_si256(a, b); + assert_eq!(r, 1); + } + test_mm256_testc_si256(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testnzc_si256() { + let a = _mm256_setr_epi64x(1, 2, 3, 4); + let b = _mm256_setr_epi64x(5, 6, 7, 8); + let r = _mm256_testnzc_si256(a, b); + assert_eq!(r, 1); + let a = _mm256_setr_epi64x(0, 0, 0, 0); + let b = _mm256_setr_epi64x(0, 0, 0, 0); + let r = _mm256_testnzc_si256(a, b); + assert_eq!(r, 0); + } + test_mm256_testnzc_si256(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testz_pd() { + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_testz_pd(a, b); + assert_eq!(r, 1); + let a = _mm256_set1_pd(-1.); + let r = _mm256_testz_pd(a, a); + assert_eq!(r, 0); + } + test_mm256_testz_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testc_pd() { + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_testc_pd(a, b); + assert_eq!(r, 1); + let a = _mm256_set1_pd(1.); + let b = _mm256_set1_pd(-1.); + let r = _mm256_testc_pd(a, b); + assert_eq!(r, 0); + } + test_mm256_testc_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testnzc_pd() { + let a = _mm256_setr_pd(1., 2., 3., 4.); + let b = _mm256_setr_pd(5., 6., 7., 8.); + let r = _mm256_testnzc_pd(a, b); + assert_eq!(r, 0); + let a = _mm256_setr_pd(1., -1., -1., -1.); + let b = _mm256_setr_pd(-1., -1., 1., 1.); + let r = _mm256_testnzc_pd(a, b); + assert_eq!(r, 1); + } + test_mm256_testnzc_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testz_pd() { + let a = _mm_setr_pd(1., 2.); + let b = _mm_setr_pd(5., 6.); + let r = _mm_testz_pd(a, b); + assert_eq!(r, 1); + let a = _mm_set1_pd(-1.); + let r = _mm_testz_pd(a, a); + assert_eq!(r, 0); + } + test_mm_testz_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testc_pd() { + let a = _mm_setr_pd(1., 2.); + let b = _mm_setr_pd(5., 6.); + let r = _mm_testc_pd(a, b); + assert_eq!(r, 1); + let a = _mm_set1_pd(1.); + let b = _mm_set1_pd(-1.); + let r = _mm_testc_pd(a, b); + assert_eq!(r, 0); + } + test_mm_testc_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testnzc_pd() { + let a = _mm_setr_pd(1., 2.); + let b = _mm_setr_pd(5., 6.); + let r = _mm_testnzc_pd(a, b); + assert_eq!(r, 0); + let a = _mm_setr_pd(1., -1.); + let b = _mm_setr_pd(-1., -1.); + let r = _mm_testnzc_pd(a, b); + assert_eq!(r, 1); + } + test_mm_testnzc_pd(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testz_ps() { + let a = _mm256_set1_ps(1.); + let r = _mm256_testz_ps(a, a); + assert_eq!(r, 1); + let a = _mm256_set1_ps(-1.); + let r = _mm256_testz_ps(a, a); + assert_eq!(r, 0); + } + test_mm256_testz_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testc_ps() { + let a = _mm256_set1_ps(1.); + let r = _mm256_testc_ps(a, a); + assert_eq!(r, 1); + let b = _mm256_set1_ps(-1.); + let r = _mm256_testc_ps(a, b); + assert_eq!(r, 0); + } + test_mm256_testc_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm256_testnzc_ps() { + let a = _mm256_set1_ps(1.); + let r = _mm256_testnzc_ps(a, a); + assert_eq!(r, 0); + let a = _mm256_setr_ps(1., -1., -1., -1., -1., -1., -1., -1.); + let b = _mm256_setr_ps(-1., -1., 1., 1., 1., 1., 1., 1.); + let r = _mm256_testnzc_ps(a, b); + assert_eq!(r, 1); + } + test_mm256_testnzc_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testz_ps() { + let a = _mm_set1_ps(1.); + let r = _mm_testz_ps(a, a); + assert_eq!(r, 1); + let a = _mm_set1_ps(-1.); + let r = _mm_testz_ps(a, a); + assert_eq!(r, 0); + } + test_mm_testz_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testc_ps() { + let a = _mm_set1_ps(1.); + let r = _mm_testc_ps(a, a); + assert_eq!(r, 1); + let b = _mm_set1_ps(-1.); + let r = _mm_testc_ps(a, b); + assert_eq!(r, 0); + } + test_mm_testc_ps(); + + #[target_feature(enable = "avx")] + unsafe fn test_mm_testnzc_ps() { + let a = _mm_set1_ps(1.); + let r = _mm_testnzc_ps(a, a); + assert_eq!(r, 0); + let a = _mm_setr_ps(1., -1., -1., -1.); + let b = _mm_setr_ps(-1., -1., 1., 1.); + let r = _mm_testnzc_ps(a, b); + assert_eq!(r, 1); + } + test_mm_testnzc_ps(); +} + +#[target_feature(enable = "sse2")] +unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { + _mm_set_epi64x(b, a) +} + +#[track_caller] +#[target_feature(enable = "sse")] +unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256(a: __m256, b: __m256) { + let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); + if _mm256_movemask_ps(cmp) != 0b11111111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { + let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); + if _mm256_movemask_pd(cmp) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { + assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) +} + +/// Stores `T` in an unaligned address +struct Unaligned { + buf: Vec, + offset: bool, + _marker: std::marker::PhantomData, +} + +impl Unaligned { + fn new(value: T) -> Self { + // Allocate extra byte for unalignment headroom + let len = std::mem::size_of::(); + let mut buf = Vec::::with_capacity(len + 1); + // Force the address to be a non-multiple of 2, so it is as unaligned as it can get. + let offset = (buf.as_ptr() as usize % 2) == 0; + let value_ptr: *const T = &value; + unsafe { + buf.as_mut_ptr().add(offset.into()).copy_from_nonoverlapping(value_ptr.cast(), len); + } + Self { buf, offset, _marker: std::marker::PhantomData } + } + + fn as_ptr(&self) -> *const T { + unsafe { self.buf.as_ptr().add(self.offset.into()).cast() } + } + + fn as_mut_ptr(&mut self) -> *mut T { + unsafe { self.buf.as_mut_ptr().add(self.offset.into()).cast() } + } + + fn read(&self) -> T { + unsafe { self.as_ptr().read_unaligned() } + } +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs new file mode 100644 index 00000000000..80d125bb856 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx2.rs @@ -0,0 +1,1613 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=+avx2 + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::mem::transmute; + +fn main() { + assert!(is_x86_feature_detected!("avx2")); + + unsafe { + test_avx2(); + } +} + +#[target_feature(enable = "avx2")] +unsafe fn test_avx2() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx2.rs + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_abs_epi32() { + #[rustfmt::skip] + let a = _mm256_setr_epi32( + 0, 1, -1, i32::MAX, + i32::MIN, 100, -100, -32, + ); + let r = _mm256_abs_epi32(a); + #[rustfmt::skip] + let e = _mm256_setr_epi32( + 0, 1, 1, i32::MAX, + i32::MAX.wrapping_add(1), 100, 100, 32, + ); + assert_eq_m256i(r, e); + } + test_mm256_abs_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_abs_epi16() { + #[rustfmt::skip] + let a = _mm256_setr_epi16( + 0, 1, -1, 2, -2, 3, -3, 4, + -4, 5, -5, i16::MAX, i16::MIN, 100, -100, -32, + ); + let r = _mm256_abs_epi16(a); + #[rustfmt::skip] + let e = _mm256_setr_epi16( + 0, 1, 1, 2, 2, 3, 3, 4, + 4, 5, 5, i16::MAX, i16::MAX.wrapping_add(1), 100, 100, 32, + ); + assert_eq_m256i(r, e); + } + test_mm256_abs_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_abs_epi8() { + #[rustfmt::skip] + let a = _mm256_setr_epi8( + 0, 1, -1, 2, -2, 3, -3, 4, + -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, + 0, 1, -1, 2, -2, 3, -3, 4, + -4, 5, -5, i8::MAX, i8::MIN, 100, -100, -32, + ); + let r = _mm256_abs_epi8(a); + #[rustfmt::skip] + let e = _mm256_setr_epi8( + 0, 1, 1, 2, 2, 3, 3, 4, + 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, + 0, 1, 1, 2, 2, 3, 3, 4, + 4, 5, 5, i8::MAX, i8::MAX.wrapping_add(1), 100, 100, 32, + ); + assert_eq_m256i(r, e); + } + test_mm256_abs_epi8(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hadd_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_hadd_epi16(a, b); + let e = _mm256_setr_epi16(4, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8); + assert_eq_m256i(r, e); + + // Test wrapping on overflow + let a = _mm256_setr_epi16( + i16::MAX, + 1, + i16::MAX, + 2, + i16::MAX, + 3, + i16::MAX, + 4, + i16::MAX, + 5, + i16::MAX, + 6, + i16::MAX, + 7, + i16::MAX, + 8, + ); + let b = _mm256_setr_epi16( + i16::MIN, + -1, + i16::MIN, + -2, + i16::MIN, + -3, + i16::MIN, + -4, + i16::MIN, + -5, + i16::MIN, + -6, + i16::MIN, + -7, + i16::MIN, + -8, + ); + let expected = _mm256_setr_epi16( + i16::MIN, + i16::MIN + 1, + i16::MIN + 2, + i16::MIN + 3, + i16::MAX, + i16::MAX - 1, + i16::MAX - 2, + i16::MAX - 3, + i16::MIN + 4, + i16::MIN + 5, + i16::MIN + 6, + i16::MIN + 7, + i16::MAX - 4, + i16::MAX - 5, + i16::MAX - 6, + i16::MAX - 7, + ); + let r = _mm256_hadd_epi16(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hadd_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hadd_epi32() { + let a = _mm256_set1_epi32(2); + let b = _mm256_set1_epi32(4); + let r = _mm256_hadd_epi32(a, b); + let e = _mm256_setr_epi32(4, 4, 8, 8, 4, 4, 8, 8); + assert_eq_m256i(r, e); + + // Test wrapping on overflow + let a = _mm256_setr_epi32(i32::MAX, 1, i32::MAX, 2, i32::MAX, 3, i32::MAX, 4); + let b = _mm256_setr_epi32(i32::MIN, -1, i32::MIN, -2, i32::MIN, -3, i32::MIN, -4); + let expected = _mm256_setr_epi32( + i32::MIN, + i32::MIN + 1, + i32::MAX, + i32::MAX - 1, + i32::MIN + 2, + i32::MIN + 3, + i32::MAX - 2, + i32::MAX - 3, + ); + let r = _mm256_hadd_epi32(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hadd_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hadds_epi16() { + let a = _mm256_set1_epi16(2); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, 1); + let b = _mm256_set1_epi16(4); + let r = _mm256_hadds_epi16(a, b); + let e = _mm256_setr_epi16(0x7FFF, 4, 4, 4, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8); + assert_eq_m256i(r, e); + + // Test saturating on overflow + let a = _mm256_setr_epi16( + i16::MAX, + 1, + i16::MAX, + 2, + i16::MAX, + 3, + i16::MAX, + 4, + i16::MAX, + 5, + i16::MAX, + 6, + i16::MAX, + 7, + i16::MAX, + 8, + ); + let b = _mm256_setr_epi16( + i16::MIN, + -1, + i16::MIN, + -2, + i16::MIN, + -3, + i16::MIN, + -4, + i16::MIN, + -5, + i16::MIN, + -6, + i16::MIN, + -7, + i16::MIN, + -8, + ); + let expected = _mm256_setr_epi16( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let r = _mm256_hadds_epi16(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hadds_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hsub_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_hsub_epi16(a, b); + let e = _mm256_set1_epi16(0); + assert_eq_m256i(r, e); + + // Test wrapping on overflow + let a = _mm256_setr_epi16( + i16::MAX, + -1, + i16::MAX, + -2, + i16::MAX, + -3, + i16::MAX, + -4, + i16::MAX, + -5, + i16::MAX, + -6, + i16::MAX, + -7, + i16::MAX, + -8, + ); + let b = _mm256_setr_epi16( + i16::MIN, + 1, + i16::MIN, + 2, + i16::MIN, + 3, + i16::MIN, + 4, + i16::MIN, + 5, + i16::MIN, + 6, + i16::MIN, + 7, + i16::MIN, + 8, + ); + let expected = _mm256_setr_epi16( + i16::MIN, + i16::MIN + 1, + i16::MIN + 2, + i16::MIN + 3, + i16::MAX, + i16::MAX - 1, + i16::MAX - 2, + i16::MAX - 3, + i16::MIN + 4, + i16::MIN + 5, + i16::MIN + 6, + i16::MIN + 7, + i16::MAX - 4, + i16::MAX - 5, + i16::MAX - 6, + i16::MAX - 7, + ); + let r = _mm256_hsub_epi16(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hsub_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hsub_epi32() { + let a = _mm256_set1_epi32(2); + let b = _mm256_set1_epi32(4); + let r = _mm256_hsub_epi32(a, b); + let e = _mm256_set1_epi32(0); + assert_eq_m256i(r, e); + + // Test wrapping on overflow + let a = _mm256_setr_epi32(i32::MAX, -1, i32::MAX, -2, i32::MAX, -3, i32::MAX, -4); + let b = _mm256_setr_epi32(i32::MIN, 1, i32::MIN, 2, i32::MIN, 3, i32::MIN, 4); + let expected = _mm256_setr_epi32( + i32::MIN, + i32::MIN + 1, + i32::MAX, + i32::MAX - 1, + i32::MIN + 2, + i32::MIN + 3, + i32::MAX - 2, + i32::MAX - 3, + ); + let r = _mm256_hsub_epi32(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hsub_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_hsubs_epi16() { + let a = _mm256_set1_epi16(2); + let a = _mm256_insert_epi16::<0>(a, 0x7fff); + let a = _mm256_insert_epi16::<1>(a, -1); + let b = _mm256_set1_epi16(4); + let r = _mm256_hsubs_epi16(a, b); + let e = _mm256_insert_epi16::<0>(_mm256_set1_epi16(0), 0x7FFF); + assert_eq_m256i(r, e); + + // Test saturating on overflow + let a = _mm256_setr_epi16( + i16::MAX, + -1, + i16::MAX, + -2, + i16::MAX, + -3, + i16::MAX, + -4, + i16::MAX, + -5, + i16::MAX, + -6, + i16::MAX, + -7, + i16::MAX, + -8, + ); + let b = _mm256_setr_epi16( + i16::MIN, + 1, + i16::MIN, + 2, + i16::MIN, + 3, + i16::MIN, + 4, + i16::MIN, + 5, + i16::MIN, + 6, + i16::MIN, + 7, + i16::MIN, + 8, + ); + let expected = _mm256_setr_epi16( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let r = _mm256_hsubs_epi16(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_hsubs_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i32gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm_i32gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); + } + test_mm_i32gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i32gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm_mask_i32gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm_setr_epi32(-1, -1, -1, 0), + ); + assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); + } + test_mm_mask_i32gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i32gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = + _mm256_i32gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); + assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); + } + test_mm256_i32gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i32gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm256_mask_i32gather_epi32::<4>( + _mm256_set1_epi32(256), + arr.as_ptr(), + _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), + _mm256_setr_epi32(-1, -1, -1, 0, 0, 0, 0, 0), + ); + assert_eq_m256i(r, _mm256_setr_epi32(0, 16, 64, 256, 256, 256, 256, 256)); + } + test_mm256_mask_i32gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i32gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm_i32gather_ps::<4>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); + } + test_mm_i32gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i32gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm_mask_i32gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), + ); + assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); + } + test_mm_mask_i32gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i32gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = + _mm256_i32gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi32(0, 16, 32, 48, 1, 2, 3, 4)); + assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 32.0, 48.0, 1.0, 2.0, 3.0, 4.0)); + } + test_mm256_i32gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i32gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm256_mask_i32gather_ps::<4>( + _mm256_set1_ps(256.0), + arr.as_ptr(), + _mm256_setr_epi32(0, 16, 64, 96, 0, 0, 0, 0), + _mm256_setr_ps(-1.0, -1.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0), + ); + assert_eq_m256(r, _mm256_setr_ps(0.0, 16.0, 64.0, 256.0, 256.0, 256.0, 256.0, 256.0)); + } + test_mm256_mask_i32gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i32gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); + assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); + } + test_mm_i32gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i32gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm_mask_i32gather_epi64::<8>( + _mm_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi32(16, 16, 16, 16), + _mm_setr_epi64x(-1, 0), + ); + assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); + } + test_mm_mask_i32gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i32gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm256_i32gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); + } + test_mm256_i32gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i32gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm256_mask_i32gather_epi64::<8>( + _mm256_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm256_setr_epi64x(-1, -1, -1, 0), + ); + assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); + } + test_mm256_mask_i32gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i32gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 0, 0)); + assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); + } + test_mm_i32gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i32gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm_mask_i32gather_pd::<8>( + _mm_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi32(16, 16, 16, 16), + _mm_setr_pd(-1.0, 0.0), + ); + assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); + } + test_mm_mask_i32gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i32gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm256_i32gather_pd::<8>(arr.as_ptr(), _mm_setr_epi32(0, 16, 32, 48)); + assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); + } + test_mm256_i32gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i32gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm256_mask_i32gather_pd::<8>( + _mm256_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi32(0, 16, 64, 96), + _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), + ); + assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); + } + test_mm256_mask_i32gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i64gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm_i64gather_epi32::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + assert_eq_m128i(r, _mm_setr_epi32(0, 16, 0, 0)); + } + test_mm_i64gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i64gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm_mask_i64gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm_setr_epi64x(0, 16), + _mm_setr_epi32(-1, 0, -1, 0), + ); + assert_eq_m128i(r, _mm_setr_epi32(0, 256, 0, 0)); + } + test_mm_mask_i64gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i64gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm256_i64gather_epi32::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + assert_eq_m128i(r, _mm_setr_epi32(0, 16, 32, 48)); + } + test_mm256_i64gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i64gather_epi32() { + let arr: [i32; 128] = core::array::from_fn(|i| i as i32); + // A multiplier of 4 is word-addressing + let r = _mm256_mask_i64gather_epi32::<4>( + _mm_set1_epi32(256), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm_setr_epi32(-1, -1, -1, 0), + ); + assert_eq_m128i(r, _mm_setr_epi32(0, 16, 64, 256)); + } + test_mm256_mask_i64gather_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i64gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm_i64gather_ps::<4>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 0.0, 0.0)); + } + test_mm_i64gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i64gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm_mask_i64gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm_setr_epi64x(0, 16), + _mm_setr_ps(-1.0, 0.0, -1.0, 0.0), + ); + assert_eq_m128(r, _mm_setr_ps(0.0, 256.0, 0.0, 0.0)); + } + test_mm_mask_i64gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i64gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm256_i64gather_ps::<4>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 32.0, 48.0)); + } + test_mm256_i64gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i64gather_ps() { + let arr: [f32; 128] = core::array::from_fn(|i| i as f32); + // A multiplier of 4 is word-addressing for f32s + let r = _mm256_mask_i64gather_ps::<4>( + _mm_set1_ps(256.0), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm_setr_ps(-1.0, -1.0, -1.0, 0.0), + ); + assert_eq_m128(r, _mm_setr_ps(0.0, 16.0, 64.0, 256.0)); + } + test_mm256_mask_i64gather_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i64gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm_i64gather_epi64::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + assert_eq_m128i(r, _mm_setr_epi64x(0, 16)); + } + test_mm_i64gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i64gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm_mask_i64gather_epi64::<8>( + _mm_set1_epi64x(256), + arr.as_ptr(), + _mm_setr_epi64x(16, 16), + _mm_setr_epi64x(-1, 0), + ); + assert_eq_m128i(r, _mm_setr_epi64x(16, 256)); + } + test_mm_mask_i64gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i64gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm256_i64gather_epi64::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 32, 48)); + } + test_mm256_i64gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i64gather_epi64() { + let arr: [i64; 128] = core::array::from_fn(|i| i as i64); + // A multiplier of 8 is word-addressing for i64s + let r = _mm256_mask_i64gather_epi64::<8>( + _mm256_set1_epi64x(256), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm256_setr_epi64x(-1, -1, -1, 0), + ); + assert_eq_m256i(r, _mm256_setr_epi64x(0, 16, 64, 256)); + } + test_mm256_mask_i64gather_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_i64gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm_i64gather_pd::<8>(arr.as_ptr(), _mm_setr_epi64x(0, 16)); + assert_eq_m128d(r, _mm_setr_pd(0.0, 16.0)); + } + test_mm_i64gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_mask_i64gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm_mask_i64gather_pd::<8>( + _mm_set1_pd(256.0), + arr.as_ptr(), + _mm_setr_epi64x(16, 16), + _mm_setr_pd(-1.0, 0.0), + ); + assert_eq_m128d(r, _mm_setr_pd(16.0, 256.0)); + } + test_mm_mask_i64gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_i64gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm256_i64gather_pd::<8>(arr.as_ptr(), _mm256_setr_epi64x(0, 16, 32, 48)); + assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 32.0, 48.0)); + } + test_mm256_i64gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mask_i64gather_pd() { + let arr: [f64; 128] = core::array::from_fn(|i| i as f64); + // A multiplier of 8 is word-addressing for f64s + let r = _mm256_mask_i64gather_pd::<8>( + _mm256_set1_pd(256.0), + arr.as_ptr(), + _mm256_setr_epi64x(0, 16, 64, 96), + _mm256_setr_pd(-1.0, -1.0, -1.0, 0.0), + ); + assert_eq_m256d(r, _mm256_setr_pd(0.0, 16.0, 64.0, 256.0)); + } + test_mm256_mask_i64gather_pd(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_madd_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_madd_epi16(a, b); + let e = _mm256_set1_epi32(16); + assert_eq_m256i(r, e); + } + test_mm256_madd_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_maddubs_epi16() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(4); + let r = _mm256_maddubs_epi16(a, b); + let e = _mm256_set1_epi16(16); + assert_eq_m256i(r, e); + } + test_mm256_maddubs_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_maskload_epi32() { + let nums = [1, 2, 3, 4]; + let a = &nums as *const i32; + let mask = _mm_setr_epi32(-1, 0, 0, -1); + let r = _mm_maskload_epi32(a, mask); + let e = _mm_setr_epi32(1, 0, 0, 4); + assert_eq_m128i(r, e); + + // Unaligned pointer + let a = Unaligned::new([1i32, 2, 3, 4]); + let mask = _mm_setr_epi32(0, !0, 0, !0); + let r = _mm_maskload_epi32(a.as_ptr().cast(), mask); + let e = _mm_setr_epi32(0, 2, 0, 4); + assert_eq_m128i(r, e); + + // Only loading first element, so slice can be short. + let a = &[2i32]; + let mask = _mm_setr_epi32(!0, 0, 0, 0); + let r = _mm_maskload_epi32(a.as_ptr(), mask); + let e = _mm_setr_epi32(2, 0, 0, 0); + assert_eq_m128i(r, e); + + // Only loading last element, so slice can be short. + let a = &[2i32]; + let mask = _mm_setr_epi32(0, 0, 0, !0); + let r = _mm_maskload_epi32(a.as_ptr().wrapping_sub(3), mask); + let e = _mm_setr_epi32(0, 0, 0, 2); + assert_eq_m128i(r, e); + } + test_mm_maskload_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_maskload_epi32() { + let nums = [1, 2, 3, 4, 5, 6, 7, 8]; + let a = &nums as *const i32; + let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); + let r = _mm256_maskload_epi32(a, mask); + let e = _mm256_setr_epi32(1, 0, 0, 4, 0, 6, 7, 0); + assert_eq_m256i(r, e); + + // Unaligned pointer + let a = Unaligned::new([1i32, 2, 3, 4, 5, 6, 7, 8]); + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let r = _mm256_maskload_epi32(a.as_ptr().cast(), mask); + let e = _mm256_setr_epi32(0, 2, 0, 4, 0, 6, 0, 8); + assert_eq_m256i(r, e); + + // Only loading first element, so slice can be short. + let a = &[2i32]; + let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); + let r = _mm256_maskload_epi32(a.as_ptr(), mask); + let e = _mm256_setr_epi32(2, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m256i(r, e); + + // Only loading last element, so slice can be short. + let a = &[2i32]; + let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); + let r = _mm256_maskload_epi32(a.as_ptr().wrapping_sub(7), mask); + let e = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, 2); + assert_eq_m256i(r, e); + } + test_mm256_maskload_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_maskload_epi64() { + let nums = [1_i64, 2_i64]; + let a = &nums as *const i64; + let mask = _mm_setr_epi64x(0, -1); + let r = _mm_maskload_epi64(a, mask); + let e = _mm_setr_epi64x(0, 2); + assert_eq_m128i(r, e); + + // Unaligned pointer + let a = Unaligned::new([1i64, 2]); + let mask = _mm_setr_epi64x(0, !0); + let r = _mm_maskload_epi64(a.as_ptr().cast(), mask); + let e = _mm_setr_epi64x(0, 2); + assert_eq_m128i(r, e); + + // Only loading first element, so slice can be short. + let a = &[2i64]; + let mask = _mm_setr_epi64x(!0, 0); + let r = _mm_maskload_epi64(a.as_ptr(), mask); + let e = _mm_setr_epi64x(2, 0); + assert_eq_m128i(r, e); + + // Only loading last element, so slice can be short. + let a = &[2i64]; + let mask = _mm_setr_epi64x(0, !0); + let r = _mm_maskload_epi64(a.as_ptr().wrapping_sub(1), mask); + let e = _mm_setr_epi64x(0, 2); + assert_eq_m128i(r, e); + } + test_mm_maskload_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_maskload_epi64() { + let nums = [1_i64, 2_i64, 3_i64, 4_i64]; + let a = &nums as *const i64; + let mask = _mm256_setr_epi64x(0, -1, -1, 0); + let r = _mm256_maskload_epi64(a, mask); + let e = _mm256_setr_epi64x(0, 2, 3, 0); + assert_eq_m256i(r, e); + + // Unaligned pointer + let a = Unaligned::new([1i64, 2, 3, 4]); + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let r = _mm256_maskload_epi64(a.as_ptr().cast(), mask); + let e = _mm256_setr_epi64x(0, 2, 0, 4); + assert_eq_m256i(r, e); + + // Only loading first element, so slice can be short. + let a = &[2i64]; + let mask = _mm256_setr_epi64x(!0, 0, 0, 0); + let r = _mm256_maskload_epi64(a.as_ptr(), mask); + let e = _mm256_setr_epi64x(2, 0, 0, 0); + assert_eq_m256i(r, e); + + // Only loading last element, so slice can be short. + let a = &[2i64]; + let mask = _mm256_setr_epi64x(0, 0, 0, !0); + let r = _mm256_maskload_epi64(a.as_ptr().wrapping_sub(3), mask); + let e = _mm256_setr_epi64x(0, 0, 0, 2); + assert_eq_m256i(r, e); + } + test_mm256_maskload_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_maskstore_epi32() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let mut arr = [-1, -1, -1, -1]; + let mask = _mm_setr_epi32(-1, 0, 0, -1); + _mm_maskstore_epi32(arr.as_mut_ptr(), mask, a); + let e = [1, -1, -1, 4]; + assert_eq!(arr, e); + + // Unaligned pointer + let mut r = Unaligned::new([0i32; 4]); + let mask = _mm_setr_epi32(0, !0, 0, !0); + let a = _mm_setr_epi32(1, 2, 3, 4); + _mm_maskstore_epi32(r.as_mut_ptr().cast(), mask, a); + let e = [0i32, 2, 0, 4]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0i32]; + let mask = _mm_setr_epi32(!0, 0, 0, 0); + let a = _mm_setr_epi32(1, 2, 3, 4); + _mm_maskstore_epi32(r.as_mut_ptr(), mask, a); + let e = [1i32]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0i32]; + let mask = _mm_setr_epi32(0, 0, 0, !0); + let a = _mm_setr_epi32(1, 2, 3, 4); + _mm_maskstore_epi32(r.as_mut_ptr().wrapping_sub(3), mask, a); + let e = [4i32]; + assert_eq!(r, e); + } + test_mm_maskstore_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_maskstore_epi32() { + let a = _mm256_setr_epi32(1, 0x6d726f, 3, 42, 0x777161, 6, 7, 8); + let mut arr = [-1, -1, -1, 0x776173, -1, 0x68657265, -1, -1]; + let mask = _mm256_setr_epi32(-1, 0, 0, -1, 0, -1, -1, 0); + _mm256_maskstore_epi32(arr.as_mut_ptr(), mask, a); + let e = [1, -1, -1, 42, -1, 6, 7, -1]; + assert_eq!(arr, e); + + // Unaligned pointer + let mut r = Unaligned::new([0i32; 8]); + let mask = _mm256_setr_epi32(0, !0, 0, !0, 0, !0, 0, !0); + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + _mm256_maskstore_epi32(r.as_mut_ptr().cast(), mask, a); + let e = [0i32, 2, 0, 4, 0, 6, 0, 8]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0i32]; + let mask = _mm256_setr_epi32(!0, 0, 0, 0, 0, 0, 0, 0); + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + _mm256_maskstore_epi32(r.as_mut_ptr(), mask, a); + let e = [1i32]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0i32]; + let mask = _mm256_setr_epi32(0, 0, 0, 0, 0, 0, 0, !0); + let a = _mm256_setr_epi32(1, 2, 3, 4, 5, 6, 7, 8); + _mm256_maskstore_epi32(r.as_mut_ptr().wrapping_sub(7), mask, a); + let e = [8i32]; + assert_eq!(r, e); + } + test_mm256_maskstore_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_maskstore_epi64() { + let a = _mm_setr_epi64x(1_i64, 2_i64); + let mut arr = [-1_i64, -1_i64]; + let mask = _mm_setr_epi64x(0, -1); + _mm_maskstore_epi64(arr.as_mut_ptr(), mask, a); + let e = [-1, 2]; + assert_eq!(arr, e); + + // Unaligned pointer + let mut r = Unaligned::new([0i64; 2]); + let mask = _mm_setr_epi64x(0, !0); + let a = _mm_setr_epi64x(1, 2); + _mm_maskstore_epi64(r.as_mut_ptr().cast(), mask, a); + let e = [0i64, 2]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0i64]; + let mask = _mm_setr_epi64x(!0, 0); + let a = _mm_setr_epi64x(1, 2); + _mm_maskstore_epi64(r.as_mut_ptr(), mask, a); + let e = [1i64]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0i64]; + let mask = _mm_setr_epi64x(0, !0); + let a = _mm_setr_epi64x(1, 2); + _mm_maskstore_epi64(r.as_mut_ptr().wrapping_sub(1), mask, a); + let e = [2i64]; + assert_eq!(r, e); + } + test_mm_maskstore_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_maskstore_epi64() { + let a = _mm256_setr_epi64x(1_i64, 2_i64, 3_i64, 4_i64); + let mut arr = [-1_i64, -1_i64, -1_i64, -1_i64]; + let mask = _mm256_setr_epi64x(0, -1, -1, 0); + _mm256_maskstore_epi64(arr.as_mut_ptr(), mask, a); + let e = [-1, 2, 3, -1]; + assert_eq!(arr, e); + + // Unaligned pointer + let mut r = Unaligned::new([0i64; 4]); + let mask = _mm256_setr_epi64x(0, !0, 0, !0); + let a = _mm256_setr_epi64x(1, 2, 3, 4); + _mm256_maskstore_epi64(r.as_mut_ptr().cast(), mask, a); + let e = [0i64, 2, 0, 4]; + assert_eq!(r.read(), e); + + // Only storing first element, so slice can be short. + let mut r = [0i64]; + let mask = _mm256_setr_epi64x(!0, 0, 0, 0); + let a = _mm256_setr_epi64x(1, 2, 3, 4); + _mm256_maskstore_epi64(r.as_mut_ptr(), mask, a); + let e = [1i64]; + assert_eq!(r, e); + + // Only storing last element, so slice can be short. + let mut r = [0i64]; + let mask = _mm256_setr_epi64x(0, 0, 0, !0); + let a = _mm256_setr_epi64x(1, 2, 3, 4); + _mm256_maskstore_epi64(r.as_mut_ptr().wrapping_sub(3), mask, a); + let e = [4i64]; + assert_eq!(r, e); + } + test_mm256_maskstore_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mpsadbw_epu8() { + let a = _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 2, 4, 6, 8, 10, 12, 14, 16, + 18, 20, 22, 24, 26, 28, 30, + ); + + let r = _mm256_mpsadbw_epu8::<0b000>(a, a); + let e = _mm256_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28, 0, 8, 16, 24, 32, 40, 48, 56); + assert_eq_m256i(r, e); + + let r = _mm256_mpsadbw_epu8::<0b001>(a, a); + let e = _mm256_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12, 32, 24, 16, 8, 0, 8, 16, 24); + assert_eq_m256i(r, e); + + let r = _mm256_mpsadbw_epu8::<0b100>(a, a); + let e = _mm256_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44, 32, 40, 48, 56, 64, 72, 80, 88); + assert_eq_m256i(r, e); + + let r = _mm256_mpsadbw_epu8::<0b101>(a, a); + let e = _mm256_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28, 0, 8, 16, 24, 32, 40, 48, 56); + assert_eq_m256i(r, e); + + let r = _mm256_mpsadbw_epu8::<0b111>(a, a); + let e = _mm256_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4, 64, 56, 48, 40, 32, 24, 16, 8); + assert_eq_m256i(r, e); + } + test_mm256_mpsadbw_epu8(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_mulhrs_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_mullo_epi16(a, b); + let e = _mm256_set1_epi16(8); + assert_eq_m256i(r, e); + } + test_mm256_mulhrs_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_packs_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_packs_epi16(a, b); + #[rustfmt::skip] + let e = _mm256_setr_epi8( + 2, 2, 2, 2, 2, 2, 2, 2, + 4, 4, 4, 4, 4, 4, 4, 4, + 2, 2, 2, 2, 2, 2, 2, 2, + 4, 4, 4, 4, 4, 4, 4, 4, + ); + + assert_eq_m256i(r, e); + } + test_mm256_packs_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_packs_epi32() { + let a = _mm256_set1_epi32(2); + let b = _mm256_set1_epi32(4); + let r = _mm256_packs_epi32(a, b); + let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4); + + assert_eq_m256i(r, e); + } + test_mm256_packs_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_packus_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(4); + let r = _mm256_packus_epi16(a, b); + #[rustfmt::skip] + let e = _mm256_setr_epi8( + 2, 2, 2, 2, 2, 2, 2, 2, + 4, 4, 4, 4, 4, 4, 4, 4, + 2, 2, 2, 2, 2, 2, 2, 2, + 4, 4, 4, 4, 4, 4, 4, 4, + ); + + assert_eq_m256i(r, e); + } + test_mm256_packus_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_packus_epi32() { + let a = _mm256_set1_epi32(2); + let b = _mm256_set1_epi32(4); + let r = _mm256_packus_epi32(a, b); + let e = _mm256_setr_epi16(2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2, 4, 4, 4, 4); + + assert_eq_m256i(r, e); + } + test_mm256_packus_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_permutevar8x32_epi32() { + let a = _mm256_setr_epi32(100, 200, 300, 400, 500, 600, 700, 800); + let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4); + let expected = _mm256_setr_epi32(600, 100, 600, 200, 800, 700, 400, 500); + let r = _mm256_permutevar8x32_epi32(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_permutevar8x32_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_permute2x128_si256() { + let a = _mm256_setr_epi64x(100, 200, 500, 600); + let b = _mm256_setr_epi64x(300, 400, 700, 800); + let r = _mm256_permute2x128_si256::<0b00_01_00_11>(a, b); + let e = _mm256_setr_epi64x(700, 800, 500, 600); + assert_eq_m256i(r, e); + } + test_mm256_permute2x128_si256(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_permutevar8x32_ps() { + let a = _mm256_setr_ps(1., 2., 3., 4., 5., 6., 7., 8.); + let b = _mm256_setr_epi32(5, 0, 5, 1, 7, 6, 3, 4); + let r = _mm256_permutevar8x32_ps(a, b); + let e = _mm256_setr_ps(6., 1., 6., 2., 8., 7., 4., 5.); + assert_eq_m256(r, e); + } + test_mm256_permutevar8x32_ps(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sad_epu8() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(4); + let r = _mm256_sad_epu8(a, b); + let e = _mm256_set1_epi64x(16); + assert_eq_m256i(r, e); + } + test_mm256_sad_epu8(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_shuffle_epi8() { + #[rustfmt::skip] + let a = _mm256_setr_epi8( + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ); + #[rustfmt::skip] + let b = _mm256_setr_epi8( + 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, + 12, 5, 5, 10, 4, 1, 8, 0, + 4, 128u8 as i8, 4, 3, 24, 12, 6, 19, + 12, 5, 5, 10, 4, 1, 8, 0, + ); + #[rustfmt::skip] + let expected = _mm256_setr_epi8( + 5, 0, 5, 4, 9, 13, 7, 4, + 13, 6, 6, 11, 5, 2, 9, 1, + 21, 0, 21, 20, 25, 29, 23, 20, + 29, 22, 22, 27, 21, 18, 25, 17, + ); + let r = _mm256_shuffle_epi8(a, b); + assert_eq_m256i(r, expected); + } + test_mm256_shuffle_epi8(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sign_epi16() { + let a = _mm256_set1_epi16(2); + let b = _mm256_set1_epi16(-1); + let r = _mm256_sign_epi16(a, b); + let e = _mm256_set1_epi16(-2); + assert_eq_m256i(r, e); + } + test_mm256_sign_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sign_epi32() { + let a = _mm256_set1_epi32(2); + let b = _mm256_set1_epi32(-1); + let r = _mm256_sign_epi32(a, b); + let e = _mm256_set1_epi32(-2); + assert_eq_m256i(r, e); + } + test_mm256_sign_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sign_epi8() { + let a = _mm256_set1_epi8(2); + let b = _mm256_set1_epi8(-1); + let r = _mm256_sign_epi8(a, b); + let e = _mm256_set1_epi8(-2); + assert_eq_m256i(r, e); + } + test_mm256_sign_epi8(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sll_epi16() { + let a = _mm256_setr_epi16( + 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, + -0xEE, 0xFF, -0xFF, + ); + let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi16( + 0x880, -0x880, 0x990, -0x990, 0xAA0, -0xAA0, 0xBB0, -0xBB0, 0xCC0, -0xCC0, 0xDD0, + -0xDD0, 0xEE0, -0xEE0, 0xFF0, -0xFF0, + ), + ); + let r = _mm256_sll_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m256i(r, _mm256_set1_epi16(0)); + let r = _mm256_sll_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi16(0)); + } + test_mm256_sll_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sll_epi32() { + let a = + _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi32( + 0xCCCC0, -0xCCCC0, 0xDDDD0, -0xDDDD0, 0xEEEE0, -0xEEEE0, 0xFFFF0, -0xFFFF0, + ), + ); + let r = _mm256_sll_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m256i(r, _mm256_set1_epi32(0)); + let r = _mm256_sll_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi32(0)); + } + test_mm256_sll_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sll_epi64() { + let a = _mm256_set_epi64x(0xEEEEEEEE, -0xEEEEEEEE, 0xFFFFFFFF, -0xFFFFFFFF); + let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i(r, _mm256_set_epi64x(0xEEEEEEEE0, -0xEEEEEEEE0, 0xFFFFFFFF0, -0xFFFFFFFF0)); + let r = _mm256_sll_epi64(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, 64)); + assert_eq_m256i(r, _mm256_set1_epi64x(0)); + let r = _mm256_sll_epi64(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi64x(0)); + } + test_mm256_sll_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sra_epi16() { + let a = _mm256_setr_epi16( + 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, + -0xEE, 0xFF, -0xFF, + ); + let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi16( + 0x8, -0x9, 0x9, -0xA, 0xA, -0xB, 0xB, -0xC, 0xC, -0xD, 0xD, -0xE, 0xE, -0xF, 0xF, + -0x10, + ), + ); + let r = _mm256_sra_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m256i( + r, + _mm256_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1), + ); + let r = _mm256_sra_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i( + r, + _mm256_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1), + ); + } + test_mm256_sra_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sra_epi32() { + let a = + _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi32(0xCCC, -0xCCD, 0xDDD, -0xDDE, 0xEEE, -0xEEF, 0xFFF, -0x1000), + ); + let r = _mm256_sra_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m256i(r, _mm256_setr_epi32(0, -1, 0, -1, 0, -1, 0, -1)); + let r = _mm256_sra_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_setr_epi32(0, -1, 0, -1, 0, -1, 0, -1)); + } + test_mm256_sra_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srl_epi16() { + let a = _mm256_setr_epi16( + 0x88, -0x88, 0x99, -0x99, 0xAA, -0xAA, 0xBB, -0xBB, 0xCC, -0xCC, 0xDD, -0xDD, 0xEE, + -0xEE, 0xFF, -0xFF, + ); + let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi16( + 0x8, 0xFF7, 0x9, 0xFF6, 0xA, 0xFF5, 0xB, 0xFF4, 0xC, 0xFF3, 0xD, 0xFF2, 0xE, 0xFF1, + 0xF, 0xFF0, + ), + ); + let r = _mm256_srl_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m256i(r, _mm256_set1_epi16(0)); + let r = _mm256_srl_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi16(0)); + } + test_mm256_srl_epi16(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srl_epi32() { + let a = + _mm256_setr_epi32(0xCCCC, -0xCCCC, 0xDDDD, -0xDDDD, 0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_setr_epi32( + 0xCCC, 0xFFFF333, 0xDDD, 0xFFFF222, 0xEEE, 0xFFFF111, 0xFFF, 0xFFFF000, + ), + ); + let r = _mm256_srl_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m256i(r, _mm256_set1_epi32(0)); + let r = _mm256_srl_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi32(0)); + } + test_mm256_srl_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srl_epi64() { + let a = _mm256_set_epi64x(0xEEEEEEEE, -0xEEEEEEEE, 0xFFFFFFFF, -0xFFFFFFFF); + let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, 4)); + assert_eq_m256i( + r, + _mm256_set_epi64x(0xEEEEEEE, 0xFFFFFFFF1111111, 0xFFFFFFF, 0xFFFFFFFF0000000), + ); + let r = _mm256_srl_epi64(a, _mm_set_epi64x(4, 0)); + assert_eq_m256i(r, a); + let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, 64)); + assert_eq_m256i(r, _mm256_set1_epi64x(0)); + let r = _mm256_srl_epi64(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m256i(r, _mm256_set1_epi64x(0)); + } + test_mm256_srl_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_sllv_epi32() { + let a = _mm_set_epi32(1, 2, 3, 4); + let b = _mm_set_epi32(4, 3, 2, 1); + let r = _mm_sllv_epi32(a, b); + let e = _mm_set_epi32(16, 16, 12, 8); + assert_eq_m128i(r, e); + } + test_mm_sllv_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sllv_epi32() { + let a = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let r = _mm256_sllv_epi32(a, b); + let e = _mm256_set_epi32(256, 256, 192, 128, 80, 48, 28, 16); + assert_eq_m256i(r, e); + } + test_mm256_sllv_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_sllv_epi64() { + let a = _mm_set_epi64x(2, 3); + let b = _mm_set_epi64x(1, 2); + let r = _mm_sllv_epi64(a, b); + let e = _mm_set_epi64x(4, 12); + assert_eq_m128i(r, e); + } + test_mm_sllv_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_sllv_epi64() { + let a = _mm256_set_epi64x(1, 2, 3, 4); + let b = _mm256_set_epi64x(4, 3, 2, 1); + let r = _mm256_sllv_epi64(a, b); + let e = _mm256_set_epi64x(16, 16, 12, 8); + assert_eq_m256i(r, e); + } + test_mm256_sllv_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_srav_epi32() { + let a = _mm_set_epi32(16, -32, 64, -128); + let b = _mm_set_epi32(4, 3, 2, 1); + let r = _mm_srav_epi32(a, b); + let e = _mm_set_epi32(1, -4, 16, -64); + assert_eq_m128i(r, e); + } + test_mm_srav_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srav_epi32() { + let a = _mm256_set_epi32(256, -512, 1024, -2048, 4096, -8192, 16384, -32768); + let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let r = _mm256_srav_epi32(a, b); + let e = _mm256_set_epi32(1, -4, 16, -64, 256, -1024, 4096, -16384); + assert_eq_m256i(r, e); + } + test_mm256_srav_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_srlv_epi32() { + let a = _mm_set_epi32(16, 32, 64, 128); + let b = _mm_set_epi32(4, 3, 2, 1); + let r = _mm_srlv_epi32(a, b); + let e = _mm_set_epi32(1, 4, 16, 64); + assert_eq_m128i(r, e); + } + test_mm_srlv_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srlv_epi32() { + let a = _mm256_set_epi32(256, 512, 1024, 2048, 4096, 8192, 16384, 32768); + let b = _mm256_set_epi32(8, 7, 6, 5, 4, 3, 2, 1); + let r = _mm256_srlv_epi32(a, b); + let e = _mm256_set_epi32(1, 4, 16, 64, 256, 1024, 4096, 16384); + assert_eq_m256i(r, e); + } + test_mm256_srlv_epi32(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm_srlv_epi64() { + let a = _mm_set_epi64x(4, 8); + let b = _mm_set_epi64x(2, 1); + let r = _mm_srlv_epi64(a, b); + let e = _mm_set_epi64x(1, 4); + assert_eq_m128i(r, e); + } + test_mm_srlv_epi64(); + + #[target_feature(enable = "avx2")] + unsafe fn test_mm256_srlv_epi64() { + let a = _mm256_set_epi64x(16, 32, 64, 128); + let b = _mm256_set_epi64x(4, 3, 2, 1); + let r = _mm256_srlv_epi64(a, b); + let e = _mm256_set_epi64x(1, 4, 16, 64); + assert_eq_m256i(r, e); + } + test_mm256_srlv_epi64(); +} + +#[target_feature(enable = "sse2")] +unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { + _mm_set_epi64x(b, a) +} + +#[track_caller] +#[target_feature(enable = "sse")] +unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256(a: __m256, b: __m256) { + let cmp = _mm256_cmp_ps::<_CMP_EQ_OQ>(a, b); + if _mm256_movemask_ps(cmp) != 0b11111111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256d(a: __m256d, b: __m256d) { + let cmp = _mm256_cmp_pd::<_CMP_EQ_OQ>(a, b); + if _mm256_movemask_pd(cmp) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "avx")] +unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { + assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) +} + +/// Stores `T` in an unaligned address +struct Unaligned { + buf: Vec, + offset: bool, + _marker: std::marker::PhantomData, +} + +impl Unaligned { + fn new(value: T) -> Self { + // Allocate extra byte for unalignment headroom + let len = std::mem::size_of::(); + let mut buf = Vec::::with_capacity(len + 1); + // Force the address to be a non-multiple of 2, so it is as unaligned as it can get. + let offset = (buf.as_ptr() as usize % 2) == 0; + let value_ptr: *const T = &value; + unsafe { + buf.as_mut_ptr().add(offset.into()).copy_from_nonoverlapping(value_ptr.cast(), len); + } + Self { buf, offset, _marker: std::marker::PhantomData } + } + + fn as_ptr(&self) -> *const T { + unsafe { self.buf.as_ptr().add(self.offset.into()).cast() } + } + + fn as_mut_ptr(&mut self) -> *mut T { + unsafe { self.buf.as_mut_ptr().add(self.offset.into()).cast() } + } + + fn read(&self) -> T { + unsafe { self.as_ptr().read_unaligned() } + } +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs new file mode 100644 index 00000000000..66bfcb20f1c --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-avx512.rs @@ -0,0 +1,217 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=+avx512f,+avx512vl,+avx512bitalg,+avx512vpopcntdq + +#![feature(avx512_target_feature)] +#![feature(stdarch_x86_avx512)] + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::mem::transmute; + +fn main() { + assert!(is_x86_feature_detected!("avx512f")); + assert!(is_x86_feature_detected!("avx512vl")); + assert!(is_x86_feature_detected!("avx512bitalg")); + assert!(is_x86_feature_detected!("avx512vpopcntdq")); + + unsafe { + test_avx512bitalg(); + test_avx512vpopcntdq(); + } +} + +// Some of the constants in the tests below are just bit patterns. They should not +// be interpreted as integers; signedness does not make sense for them, but +// __mXXXi happens to be defined in terms of signed integers. +#[allow(overflowing_literals)] +#[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] +unsafe fn test_avx512bitalg() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx512bitalg.rs + + #[target_feature(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_popcnt_epi16() { + let test_data = _mm512_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, 0xFF_FF, -1, -100, 255, 256, 2, 4, 8, 16, 32, 64, 128, 256, 512, + 1024, 2048, + ); + let actual_result = _mm512_popcnt_epi16(test_data); + let reference_result = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 12, 8, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, + ); + assert_eq_m512i(actual_result, reference_result); + } + test_mm512_popcnt_epi16(); + + #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi16() { + let test_data = _mm256_set_epi16( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1_FF, 0x3_FF, 0x7_FF, 0xF_FF, 0x1F_FF, + 0x3F_FF, 0x7F_FF, + ); + let actual_result = _mm256_popcnt_epi16(test_data); + let reference_result = + _mm256_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assert_eq_m256i(actual_result, reference_result); + } + test_mm256_popcnt_epi16(); + + #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi16() { + let test_data = _mm_set_epi16(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F); + let actual_result = _mm_popcnt_epi16(test_data); + let reference_result = _mm_set_epi16(0, 1, 2, 3, 4, 5, 6, 7); + assert_eq_m128i(actual_result, reference_result); + } + test_mm_popcnt_epi16(); + + #[target_feature(enable = "avx512bitalg,avx512f")] + unsafe fn test_mm512_popcnt_epi8() { + let test_data = _mm512_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, 183, 154, 84, 56, 227, 189, + 140, 35, 117, 219, 169, 226, 170, 13, 22, 159, 251, 73, 121, 143, 145, 85, 91, 137, 90, + 225, 21, 249, 211, 155, 228, 70, + ); + let actual_result = _mm512_popcnt_epi8(test_data); + let reference_result = _mm512_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, 6, 4, 3, 3, 5, 6, 3, 3, 5, 6, 4, 4, 4, 3, 3, 6, 7, 3, 5, 5, 3, 4, 5, 3, 4, 4, + 3, 6, 5, 5, 4, 3, + ); + assert_eq_m512i(actual_result, reference_result); + } + test_mm512_popcnt_epi8(); + + #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi8() { + let test_data = _mm256_set_epi8( + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64, 128, 171, 206, 100, + 217, 109, 253, 190, 177, 254, 179, 215, 230, 68, 201, 172, + ); + let actual_result = _mm256_popcnt_epi8(test_data); + let reference_result = _mm256_set_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1, 1, 5, 5, 3, 5, 5, 7, 6, 4, 7, 5, 6, 5, + 2, 4, 4, + ); + assert_eq_m256i(actual_result, reference_result); + } + test_mm256_popcnt_epi8(); + + #[target_feature(enable = "avx512bitalg,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi8() { + let test_data = + _mm_set_epi8(0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, -1, 2, 4, 8, 16, 32, 64); + let actual_result = _mm_popcnt_epi8(test_data); + let reference_result = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 1, 1, 1, 1, 1, 1); + assert_eq_m128i(actual_result, reference_result); + } + test_mm_popcnt_epi8(); +} + +#[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] +unsafe fn test_avx512vpopcntdq() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/avx512vpopcntdq.rs + + #[target_feature(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_popcnt_epi32() { + let test_data = _mm512_set_epi32( + 0, + 1, + -1, + 2, + 7, + 0xFF_FE, + 0x7F_FF_FF_FF, + -100, + 0x40_00_00_00, + 103, + 371, + 552, + 432_948, + 818_826_998, + 255, + 256, + ); + let actual_result = _mm512_popcnt_epi32(test_data); + let reference_result = + _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1); + assert_eq_m512i(actual_result, reference_result); + } + test_mm512_popcnt_epi32(); + + #[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm256_popcnt_epi32() { + let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100); + let actual_result = _mm256_popcnt_epi32(test_data); + let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28); + assert_eq_m256i(actual_result, reference_result); + } + test_mm256_popcnt_epi32(); + + #[target_feature(enable = "avx512vpopcntdq,avx512f,avx512vl")] + unsafe fn test_mm_popcnt_epi32() { + let test_data = _mm_set_epi32(0, 1, -1, -100); + let actual_result = _mm_popcnt_epi32(test_data); + let reference_result = _mm_set_epi32(0, 1, 32, 28); + assert_eq_m128i(actual_result, reference_result); + } + test_mm_popcnt_epi32(); + + #[target_feature(enable = "avx512vpopcntdq,avx512f")] + unsafe fn test_mm512_popcnt_epi64() { + let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100); + let actual_result = _mm512_popcnt_epi64(test_data); + let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60); + assert_eq_m512i(actual_result, reference_result); + } + test_mm512_popcnt_epi64(); + + #[target_feature(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm256_popcnt_epi64() { + let test_data = _mm256_set_epi64x(0, 1, -1, -100); + let actual_result = _mm256_popcnt_epi64(test_data); + let reference_result = _mm256_set_epi64x(0, 1, 64, 60); + assert_eq_m256i(actual_result, reference_result); + } + test_mm256_popcnt_epi64(); + + #[target_feature(enable = "avx512vpopcntdq,avx512vl")] + unsafe fn test_mm_popcnt_epi64() { + let test_data = _mm_set_epi64x(0, 1); + let actual_result = _mm_popcnt_epi64(test_data); + let reference_result = _mm_set_epi64x(0, 1); + assert_eq_m128i(actual_result, reference_result); + let test_data = _mm_set_epi64x(-1, -100); + let actual_result = _mm_popcnt_epi64(test_data); + let reference_result = _mm_set_epi64x(64, 60); + assert_eq_m128i(actual_result, reference_result); + } + test_mm_popcnt_epi64(); +} + +#[track_caller] +unsafe fn assert_eq_m512i(a: __m512i, b: __m512i) { + assert_eq!(transmute::<_, [i32; 16]>(a), transmute::<_, [i32; 16]>(b)) +} + +#[track_caller] +unsafe fn assert_eq_m256i(a: __m256i, b: __m256i) { + assert_eq!(transmute::<_, [u64; 4]>(a), transmute::<_, [u64; 4]>(b)) +} + +#[track_caller] +unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs new file mode 100644 index 00000000000..c8b92fd5458 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-pause-without-sse2.rs @@ -0,0 +1,25 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=-sse2 + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +fn main() { + assert!(!is_x86_feature_detected!("sse2")); + + unsafe { + // This is a SSE2 intrinsic, but it behaves as a no-op when SSE2 + // is not available, so it is always safe to call. + _mm_pause(); + } +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs new file mode 100644 index 00000000000..a62a5ee3781 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse.rs @@ -0,0 +1,1107 @@ +fn main() { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + assert!(is_x86_feature_detected!("sse")); + + unsafe { + tests::test_sse(); + } + } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod tests { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + use std::f32::NAN; + use std::mem::transmute; + + macro_rules! assert_approx_eq { + ($a:expr, $b:expr, $eps:expr) => {{ + let (a, b) = (&$a, &$b); + assert!( + (*a - *b).abs() < $eps, + "assertion failed: `(left !== right)` \ + (left: `{:?}`, right: `{:?}`, expect diff: `{:?}`, real diff: `{:?}`)", + *a, + *b, + $eps, + (*a - *b).abs() + ); + }}; + } + + #[target_feature(enable = "sse")] + pub(super) unsafe fn test_sse() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86{,_64}/sse.rs + + #[target_feature(enable = "sse")] + unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } + } + + #[target_feature(enable = "sse")] + unsafe fn test_mm_add_ss() { + let a = _mm_set_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_set_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_add_ss(a, b); + assert_eq_m128(r, _mm_set_ps(-1.0, 5.0, 0.0, -15.0)); + } + test_mm_add_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sub_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_sub_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(99.0, 5.0, 0.0, -10.0)); + } + test_mm_sub_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_mul_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_mul_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(100.0, 5.0, 0.0, -10.0)); + } + test_mm_mul_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_div_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_div_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(0.01, 5.0, 0.0, -10.0)); + } + test_mm_div_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sqrt_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_sqrt_ss(a); + let e = _mm_setr_ps(2.0, 13.0, 16.0, 100.0); + assert_eq_m128(r, e); + } + test_mm_sqrt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_sqrt_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_sqrt_ps(a); + let e = _mm_setr_ps(2.0, 3.6055512, 4.0, 10.0); + assert_eq_m128(r, e); + } + test_mm_sqrt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rcp_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rcp_ss(a); + let e = _mm_setr_ps(0.24993896, 13.0, 16.0, 100.0); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + assert_approx_eq!(r[0], e[0], 2. * rel_err); + for i in 1..4 { + assert_eq!(r[i], e[i]); + } + } + test_mm_rcp_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rcp_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rcp_ps(a); + let e = _mm_setr_ps(0.24993896, 0.0769043, 0.06248474, 0.0099983215); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + for i in 0..4 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm_rcp_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rsqrt_ss() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rsqrt_ss(a); + let e = _mm_setr_ps(0.49987793, 13.0, 16.0, 100.0); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + assert_approx_eq!(r[0], e[0], 2. * rel_err); + for i in 1..4 { + assert_eq!(r[i], e[i]); + } + } + test_mm_rsqrt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_rsqrt_ps() { + let a = _mm_setr_ps(4.0, 13.0, 16.0, 100.0); + let r = _mm_rsqrt_ps(a); + let e = _mm_setr_ps(0.49987793, 0.2772827, 0.24993896, 0.099990845); + let rel_err = 0.00048828125; + + let r: [f32; 4] = transmute(r); + let e: [f32; 4] = transmute(e); + for i in 0..4 { + assert_approx_eq!(r[i], e[i], 2. * rel_err); + } + } + test_mm_rsqrt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_min_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_min_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); + } + test_mm_min_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_min_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_min_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(-100.0, 5.0, 0.0, -10.0)); + + // `_mm_min_ps` can **not** be implemented using the `simd_min` rust intrinsic because + // the semantics of `simd_min` are different to those of `_mm_min_ps` regarding handling + // of `-0.0`. + let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); + let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); + let r1: [u8; 16] = transmute(_mm_min_ps(a, b)); + let r2: [u8; 16] = transmute(_mm_min_ps(b, a)); + let a: [u8; 16] = transmute(a); + let b: [u8; 16] = transmute(b); + assert_eq!(r1, b); + assert_eq!(r2, a); + assert_ne!(a, b); // sanity check that -0.0 is actually present + } + test_mm_min_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_max_ss() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_max_ss(a, b); + assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, -10.0)); + } + test_mm_max_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_max_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_max_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(-1.0, 20.0, 0.0, -5.0)); + + // `_mm_max_ps` can **not** be implemented using the `simd_max` rust intrinsic because + // the semantics of `simd_max` are different to those of `_mm_max_ps` regarding handling + // of `-0.0`. + let a = _mm_setr_ps(-0.0, 0.0, 0.0, 0.0); + let b = _mm_setr_ps(0.0, 0.0, 0.0, 0.0); + let r1: [u8; 16] = transmute(_mm_max_ps(a, b)); + let r2: [u8; 16] = transmute(_mm_max_ps(b, a)); + let a: [u8; 16] = transmute(a); + let b: [u8; 16] = transmute(b); + assert_eq!(r1, b); + assert_eq!(r2, a); + assert_ne!(a, b); // sanity check that -0.0 is actually present + } + test_mm_max_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpeq_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(-1.0, 5.0, 6.0, 7.0); + let r: [u32; 4] = transmute(_mm_cmpeq_ss(a, b)); + let e: [u32; 4] = transmute(_mm_setr_ps(transmute(0u32), 2.0, 3.0, 4.0)); + assert_eq!(r, e); + + let b2 = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let r2: [u32; 4] = transmute(_mm_cmpeq_ss(a, b2)); + let e2: [u32; 4] = transmute(_mm_setr_ps(transmute(0xffffffffu32), 2.0, 3.0, 4.0)); + assert_eq!(r2, e2); + } + test_mm_cmpeq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmplt_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) < b.extract(0) + let c1 = 0u32; // a.extract(0) < c.extract(0) + let d1 = !0u32; // a.extract(0) < d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmplt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmplt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmplt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmplt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmple_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) <= b.extract(0) + let c1 = !0u32; // a.extract(0) <= c.extract(0) + let d1 = !0u32; // a.extract(0) <= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmple_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmple_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmple_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmple_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpgt_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) > b.extract(0) + let c1 = 0u32; // a.extract(0) > c.extract(0) + let d1 = 0u32; // a.extract(0) > d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpgt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpgt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpgt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpgt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpge_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) >= b.extract(0) + let c1 = !0u32; // a.extract(0) >= c.extract(0) + let d1 = 0u32; // a.extract(0) >= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpge_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpge_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpge_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpge_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpneq_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) != b.extract(0) + let c1 = 0u32; // a.extract(0) != c.extract(0) + let d1 = !0u32; // a.extract(0) != d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpneq_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpneq_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpneq_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpneq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnlt_ss() { + // TODO: this test is exactly the same as for `_mm_cmpge_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) >= b.extract(0) + let c1 = !0u32; // a.extract(0) >= c.extract(0) + let d1 = 0u32; // a.extract(0) >= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnlt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnlt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnlt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnlt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnle_ss() { + // TODO: this test is exactly the same as for `_mm_cmpgt_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence + // of NaNs (signaling or quiet). If so, we should add tests for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) > b.extract(0) + let c1 = 0u32; // a.extract(0) > c.extract(0) + let d1 = 0u32; // a.extract(0) > d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnle_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnle_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnle_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnle_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpngt_ss() { + // TODO: this test is exactly the same as for `_mm_cmple_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) <= b.extract(0) + let c1 = !0u32; // a.extract(0) <= c.extract(0) + let d1 = !0u32; // a.extract(0) <= d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpngt_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpngt_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpngt_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpngt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnge_ss() { + // TODO: this test is exactly the same as for `_mm_cmplt_ss`, but there + // must be a difference. It may have to do with behavior in the + // presence of NaNs (signaling or quiet). If so, we should add tests + // for those. + + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(1.0, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) < b.extract(0) + let c1 = 0u32; // a.extract(0) < c.extract(0) + let d1 = !0u32; // a.extract(0) < d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpnge_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpnge_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpnge_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpnge_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpord_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = !0u32; // a.extract(0) ord b.extract(0) + let c1 = 0u32; // a.extract(0) ord c.extract(0) + let d1 = !0u32; // a.extract(0) ord d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpord_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpord_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpord_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpord_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpunord_ss() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let b = _mm_setr_ps(0.0, 5.0, 6.0, 7.0); + let c = _mm_setr_ps(NAN, 5.0, 6.0, 7.0); + let d = _mm_setr_ps(2.0, 5.0, 6.0, 7.0); + + let b1 = 0u32; // a.extract(0) unord b.extract(0) + let c1 = !0u32; // a.extract(0) unord c.extract(0) + let d1 = 0u32; // a.extract(0) unord d.extract(0) + + let rb: [u32; 4] = transmute(_mm_cmpunord_ss(a, b)); + let eb: [u32; 4] = transmute(_mm_setr_ps(transmute(b1), 2.0, 3.0, 4.0)); + assert_eq!(rb, eb); + + let rc: [u32; 4] = transmute(_mm_cmpunord_ss(a, c)); + let ec: [u32; 4] = transmute(_mm_setr_ps(transmute(c1), 2.0, 3.0, 4.0)); + assert_eq!(rc, ec); + + let rd: [u32; 4] = transmute(_mm_cmpunord_ss(a, d)); + let ed: [u32; 4] = transmute(_mm_setr_ps(transmute(d1), 2.0, 3.0, 4.0)); + assert_eq!(rd, ed); + } + test_mm_cmpunord_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpeq_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, fls, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmpeq_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpeq_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmplt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmplt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmplt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmple_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, 4.0); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmple_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmple_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpgt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmpgt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpgt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpge_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 42.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, fls]; + let r: [u32; 4] = transmute(_mm_cmpge_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpge_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpneq_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, tru, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpneq_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpneq_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnlt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpnlt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnlt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnle_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpnle_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnle_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpngt_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpngt_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpngt_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpnge_ps() { + let a = _mm_setr_ps(10.0, 50.0, 1.0, NAN); + let b = _mm_setr_ps(15.0, 20.0, 1.0, 5.0); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, tru]; + let r: [u32; 4] = transmute(_mm_cmpnge_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpnge_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpord_ps() { + let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); + let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [tru, fls, fls, fls]; + let r: [u32; 4] = transmute(_mm_cmpord_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpord_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cmpunord_ps() { + let a = _mm_setr_ps(10.0, 50.0, NAN, NAN); + let b = _mm_setr_ps(15.0, NAN, 1.0, NAN); + let tru = !0u32; + let fls = 0u32; + + let e = [fls, tru, tru, tru]; + let r: [u32; 4] = transmute(_mm_cmpunord_ps(a, b)); + assert_eq!(r, e); + } + test_mm_cmpunord_ps(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comieq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comieq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comieq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comieq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comilt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comilt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comilt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comilt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comile_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comile_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comile_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comile_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comigt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comige_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comige_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comigt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_comineq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 1, 1]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_comineq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_comineq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_comineq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomieq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomieq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomieq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomieq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomilt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomilt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomilt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomilt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomile_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 1, 0, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomile_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomile_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomile_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomigt_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomigt_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomigt_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomigt_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomige_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[1i32, 0, 1, 0]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomige_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomige_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomige_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_ucomineq_ss() { + let aa = &[3.0f32, 12.0, 23.0, NAN]; + let bb = &[3.0f32, 47.5, 1.5, NAN]; + + let ee = &[0i32, 1, 1, 1]; + + for i in 0..4 { + let a = _mm_setr_ps(aa[i], 1.0, 2.0, 3.0); + let b = _mm_setr_ps(bb[i], 0.0, 2.0, 4.0); + + let r = _mm_ucomineq_ss(a, b); + + assert_eq!( + ee[i], r, + "_mm_ucomineq_ss({:?}, {:?}) = {}, expected: {} (i={})", + a, b, r, ee[i], i + ); + } + } + test_mm_ucomineq_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_si32() { + let inputs = &[42.0f32, -3.1, 4.0e10, 4.0e-20, NAN, 2147483500.1]; + let result = &[42i32, -3, i32::MIN, 0, i32::MIN, 2147483520]; + for i in 0..inputs.len() { + let x = _mm_setr_ps(inputs[i], 1.0, 3.0, 4.0); + let e = result[i]; + let r = _mm_cvtss_si32(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvtss_si32({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvtss_si32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvttss_si32() { + let inputs = &[ + (42.0f32, 42i32), + (-31.4, -31), + (-33.5, -33), + (-34.5, -34), + (10.999, 10), + (-5.99, -5), + (4.0e10, i32::MIN), + (4.0e-10, 0), + (NAN, i32::MIN), + (2147483500.1, 2147483520), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvttss_si32(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvttss_si32({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + test_mm_cvttss_si32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_f32() { + let a = _mm_setr_ps(312.0134, 5.0, 6.0, 7.0); + assert_eq!(_mm_cvtss_f32(a), 312.0134); + } + test_mm_cvtss_f32(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtsi32_ss() { + let inputs = &[ + (4555i32, 4555.0f32), + (322223333, 322223330.0), + (-432, -432.0), + (-322223333, -322223330.0), + ]; + + for i in 0..inputs.len() { + let (x, f) = inputs[i]; + let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); + let r = _mm_cvtsi32_ss(a, x); + let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); + assert_eq_m128(e, r); + } + } + test_mm_cvtsi32_ss(); + + // Intrinsic only available on x86_64 + #[cfg(target_arch = "x86_64")] + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtss_si64() { + let inputs = &[ + (42.0f32, 42i64), + (-31.4, -31), + (-33.5, -34), + (-34.5, -34), + (4.0e10, 40_000_000_000), + (4.0e-10, 0), + (f32::NAN, i64::MIN), + (2147483500.1, 2147483520), + (9.223371e18, 9223370937343148032), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvtss_si64(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvtss_si64({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + #[cfg(target_arch = "x86_64")] + test_mm_cvtss_si64(); + + // Intrinsic only available on x86_64 + #[cfg(target_arch = "x86_64")] + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvttss_si64() { + let inputs = &[ + (42.0f32, 42i64), + (-31.4, -31), + (-33.5, -33), + (-34.5, -34), + (10.999, 10), + (-5.99, -5), + (4.0e10, 40_000_000_000), + (4.0e-10, 0), + (f32::NAN, i64::MIN), + (2147483500.1, 2147483520), + (9.223371e18, 9223370937343148032), + (9.223372e18, i64::MIN), + ]; + for i in 0..inputs.len() { + let (xi, e) = inputs[i]; + let x = _mm_setr_ps(xi, 1.0, 3.0, 4.0); + let r = _mm_cvttss_si64(x); + assert_eq!( + e, r, + "TestCase #{} _mm_cvttss_si64({:?}) = {}, expected: {}", + i, x, r, e + ); + } + } + #[cfg(target_arch = "x86_64")] + test_mm_cvttss_si64(); + + // Intrinsic only available on x86_64 + #[cfg(target_arch = "x86_64")] + #[target_feature(enable = "sse")] + unsafe fn test_mm_cvtsi64_ss() { + let inputs = &[ + (4555i64, 4555.0f32), + (322223333, 322223330.0), + (-432, -432.0), + (-322223333, -322223330.0), + (9223372036854775807, 9.223372e18), + (-9223372036854775808, -9.223372e18), + ]; + + for i in 0..inputs.len() { + let (x, f) = inputs[i]; + let a = _mm_setr_ps(5.0, 6.0, 7.0, 8.0); + let r = _mm_cvtsi64_ss(a, x); + let e = _mm_setr_ps(f, 6.0, 7.0, 8.0); + assert_eq_m128(e, r); + } + } + #[cfg(target_arch = "x86_64")] + test_mm_cvtsi64_ss(); + + #[target_feature(enable = "sse")] + unsafe fn test_mm_movemask_ps() { + let r = _mm_movemask_ps(_mm_setr_ps(-1.0, 5.0, -5.0, 0.0)); + assert_eq!(r, 0b0101); + + let r = _mm_movemask_ps(_mm_setr_ps(-1.0, -5.0, -5.0, 0.0)); + assert_eq!(r, 0b0111); + } + test_mm_movemask_ps(); + + let x = 0i8; + _mm_prefetch(&x, _MM_HINT_T0); + _mm_prefetch(&x, _MM_HINT_T1); + _mm_prefetch(&x, _MM_HINT_T2); + _mm_prefetch(&x, _MM_HINT_NTA); + _mm_prefetch(&x, _MM_HINT_ET0); + _mm_prefetch(&x, _MM_HINT_ET1); + } +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs new file mode 100644 index 00000000000..e0088b9eb24 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse2.rs @@ -0,0 +1,848 @@ +fn main() { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + assert!(is_x86_feature_detected!("sse2")); + + unsafe { + tests::test_sse2(); + } + } +} + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod tests { + #[cfg(target_arch = "x86")] + use std::arch::x86::*; + #[cfg(target_arch = "x86_64")] + use std::arch::x86_64::*; + use std::f64::NAN; + use std::mem::transmute; + + #[target_feature(enable = "sse2")] + unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { + _mm_set_epi64x(b, a) + } + + #[target_feature(enable = "sse2")] + pub(super) unsafe fn test_sse2() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86{,_64}/sse2.rs + + unsafe fn _mm_setr_epi64x(a: i64, b: i64) -> __m128i { + _mm_set_epi64x(b, a) + } + + #[track_caller] + #[target_feature(enable = "sse")] + unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } + } + + #[track_caller] + #[target_feature(enable = "sse2")] + unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) + } + + #[track_caller] + #[target_feature(enable = "sse2")] + unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } + } + + fn test_mm_pause() { + unsafe { _mm_pause() } + } + test_mm_pause(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_avg_epu8() { + let (a, b) = (_mm_set1_epi8(3), _mm_set1_epi8(9)); + let r = _mm_avg_epu8(a, b); + assert_eq_m128i(r, _mm_set1_epi8(6)); + } + test_mm_avg_epu8(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_avg_epu16() { + let (a, b) = (_mm_set1_epi16(3), _mm_set1_epi16(9)); + let r = _mm_avg_epu16(a, b); + assert_eq_m128i(r, _mm_set1_epi16(6)); + } + test_mm_avg_epu16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_madd_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm_madd_epi16(a, b); + let e = _mm_setr_epi32(29, 81, 149, 233); + assert_eq_m128i(r, e); + + let a = + _mm_setr_epi16(i16::MAX, i16::MAX, i16::MIN, i16::MIN, i16::MIN, i16::MAX, 0, 0); + let b = + _mm_setr_epi16(i16::MAX, i16::MAX, i16::MIN, i16::MIN, i16::MAX, i16::MIN, 0, 0); + let r = _mm_madd_epi16(a, b); + let e = _mm_setr_epi32(0x7FFE0002, i32::MIN, -0x7FFF0000, 0); + assert_eq_m128i(r, e); + } + test_mm_madd_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_mulhi_epi16() { + let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(-1001)); + let r = _mm_mulhi_epi16(a, b); + assert_eq_m128i(r, _mm_set1_epi16(-16)); + } + test_mm_mulhi_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_mulhi_epu16() { + let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(1001)); + let r = _mm_mulhi_epu16(a, b); + assert_eq_m128i(r, _mm_set1_epi16(15)); + } + test_mm_mulhi_epu16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_mul_epu32() { + let a = _mm_setr_epi64x(1_000_000_000, 1 << 34); + let b = _mm_setr_epi64x(1_000_000_000, 1 << 35); + let r = _mm_mul_epu32(a, b); + let e = _mm_setr_epi64x(1_000_000_000 * 1_000_000_000, 0); + assert_eq_m128i(r, e); + } + test_mm_mul_epu32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sad_epu8() { + #[rustfmt::skip] + let a = _mm_setr_epi8( + 255u8 as i8, 254u8 as i8, 253u8 as i8, 252u8 as i8, + 1, 2, 3, 4, + 155u8 as i8, 154u8 as i8, 153u8 as i8, 152u8 as i8, + 1, 2, 3, 4, + ); + let b = _mm_setr_epi8(0, 0, 0, 0, 2, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2); + let r = _mm_sad_epu8(a, b); + let e = _mm_setr_epi64x(1020, 614); + assert_eq_m128i(r, e); + } + test_mm_sad_epu8(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sll_epi16() { + let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); + let r = _mm_sll_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i( + r, + _mm_setr_epi16(0xCC0, -0xCC0, 0xDD0, -0xDD0, 0xEE0, -0xEE0, 0xFF0, -0xFF0), + ); + let r = _mm_sll_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_sll_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m128i(r, _mm_set1_epi16(0)); + let r = _mm_sll_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi16(0)); + } + test_mm_sll_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_srl_epi16() { + let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); + let r = _mm_srl_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_setr_epi16(0xC, 0xFF3, 0xD, 0xFF2, 0xE, 0xFF1, 0xF, 0xFF0)); + let r = _mm_srl_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_srl_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m128i(r, _mm_set1_epi16(0)); + let r = _mm_srl_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi16(0)); + } + test_mm_srl_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sra_epi16() { + let a = _mm_setr_epi16(0xCC, -0xCC, 0xDD, -0xDD, 0xEE, -0xEE, 0xFF, -0xFF); + let r = _mm_sra_epi16(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_setr_epi16(0xC, -0xD, 0xD, -0xE, 0xE, -0xF, 0xF, -0x10)); + let r = _mm_sra_epi16(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_sra_epi16(a, _mm_set_epi64x(0, 16)); + assert_eq_m128i(r, _mm_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1)); + let r = _mm_sra_epi16(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_setr_epi16(0, -1, 0, -1, 0, -1, 0, -1)); + } + test_mm_sra_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sll_epi32() { + let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm_sll_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_setr_epi32(0xEEEE0, -0xEEEE0, 0xFFFF0, -0xFFFF0)); + let r = _mm_sll_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_sll_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m128i(r, _mm_set1_epi32(0)); + let r = _mm_sll_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi32(0)); + } + test_mm_sll_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_srl_epi32() { + let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm_srl_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_setr_epi32(0xEEE, 0xFFFF111, 0xFFF, 0xFFFF000)); + let r = _mm_srl_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_srl_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m128i(r, _mm_set1_epi32(0)); + let r = _mm_srl_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi32(0)); + } + test_mm_srl_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sra_epi32() { + let a = _mm_setr_epi32(0xEEEE, -0xEEEE, 0xFFFF, -0xFFFF); + let r = _mm_sra_epi32(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_setr_epi32(0xEEE, -0xEEF, 0xFFF, -0x1000)); + let r = _mm_sra_epi32(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_sra_epi32(a, _mm_set_epi64x(0, 32)); + assert_eq_m128i(r, _mm_setr_epi32(0, -1, 0, -1)); + let r = _mm_sra_epi32(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_setr_epi32(0, -1, 0, -1)); + } + test_mm_sra_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sll_epi64() { + let a = _mm_set_epi64x(0xFFFFFFFF, -0xFFFFFFFF); + let r = _mm_sll_epi64(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_set_epi64x(0xFFFFFFFF0, -0xFFFFFFFF0)); + let r = _mm_sll_epi64(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_sll_epi64(a, _mm_set_epi64x(0, 64)); + assert_eq_m128i(r, _mm_set1_epi64x(0)); + let r = _mm_sll_epi64(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi64x(0)); + } + test_mm_sll_epi64(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_srl_epi64() { + let a = _mm_set_epi64x(0xFFFFFFFF, -0xFFFFFFFF); + let r = _mm_srl_epi64(a, _mm_set_epi64x(0, 4)); + assert_eq_m128i(r, _mm_set_epi64x(0xFFFFFFF, 0xFFFFFFFF0000000)); + let r = _mm_srl_epi64(a, _mm_set_epi64x(4, 0)); + assert_eq_m128i(r, a); + let r = _mm_srl_epi64(a, _mm_set_epi64x(0, 64)); + assert_eq_m128i(r, _mm_set1_epi64x(0)); + let r = _mm_srl_epi64(a, _mm_set_epi64x(0, i64::MAX)); + assert_eq_m128i(r, _mm_set1_epi64x(0)); + } + test_mm_srl_epi64(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtepi32_ps() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let r = _mm_cvtepi32_ps(a); + assert_eq_m128(r, _mm_setr_ps(1.0, 2.0, 3.0, 4.0)); + } + test_mm_cvtepi32_ps(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtps_epi32() { + let a = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let r = _mm_cvtps_epi32(a); + assert_eq_m128i(r, _mm_setr_epi32(1, 2, 3, 4)); + } + test_mm_cvtps_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvttps_epi32() { + let a = _mm_setr_ps(-1.1, 2.2, -3.3, 6.6); + let r = _mm_cvttps_epi32(a); + assert_eq_m128i(r, _mm_setr_epi32(-1, 2, -3, 6)); + + let a = _mm_setr_ps(f32::NEG_INFINITY, f32::INFINITY, f32::MIN, f32::MAX); + let r = _mm_cvttps_epi32(a); + assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, i32::MIN, i32::MIN)); + } + test_mm_cvttps_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_packs_epi16() { + let a = _mm_setr_epi16(0x80, -0x81, 0, 0, 0, 0, 0, 0); + let b = _mm_setr_epi16(0, 0, 0, 0, 0, 0, -0x81, 0x80); + let r = _mm_packs_epi16(a, b); + assert_eq_m128i( + r, + _mm_setr_epi8(0x7F, -0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -0x80, 0x7F), + ); + } + test_mm_packs_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_packus_epi16() { + let a = _mm_setr_epi16(0x100, -1, 0, 0, 0, 0, 0, 0); + let b = _mm_setr_epi16(0, 0, 0, 0, 0, 0, -1, 0x100); + let r = _mm_packus_epi16(a, b); + assert_eq_m128i(r, _mm_setr_epi8(!0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, !0)); + } + test_mm_packus_epi16(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_packs_epi32() { + let a = _mm_setr_epi32(0x8000, -0x8001, 0, 0); + let b = _mm_setr_epi32(0, 0, -0x8001, 0x8000); + let r = _mm_packs_epi32(a, b); + assert_eq_m128i(r, _mm_setr_epi16(0x7FFF, -0x8000, 0, 0, 0, 0, -0x8000, 0x7FFF)); + } + test_mm_packs_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_min_sd() { + let a = _mm_setr_pd(1.0, 2.0); + let b = _mm_setr_pd(5.0, 10.0); + let r = _mm_min_sd(a, b); + assert_eq_m128d(r, _mm_setr_pd(1.0, 2.0)); + } + test_mm_min_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_min_pd() { + let a = _mm_setr_pd(-1.0, 5.0); + let b = _mm_setr_pd(-100.0, 20.0); + let r = _mm_min_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(-100.0, 5.0)); + + // `_mm_min_pd` can **not** be implemented using the `simd_min` rust intrinsic because + // the semantics of `simd_min` are different to those of `_mm_min_pd` regarding handling + // of `-0.0`. + let a = _mm_setr_pd(-0.0, 0.0); + let b = _mm_setr_pd(0.0, 0.0); + let r1: [u8; 16] = transmute(_mm_min_pd(a, b)); + let r2: [u8; 16] = transmute(_mm_min_pd(b, a)); + let a: [u8; 16] = transmute(a); + let b: [u8; 16] = transmute(b); + assert_eq!(r1, b); + assert_eq!(r2, a); + assert_ne!(a, b); // sanity check that -0.0 is actually present + } + test_mm_min_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_max_sd() { + let a = _mm_setr_pd(1.0, 2.0); + let b = _mm_setr_pd(5.0, 10.0); + let r = _mm_max_sd(a, b); + assert_eq_m128d(r, _mm_setr_pd(5.0, 2.0)); + } + test_mm_max_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_max_pd() { + let a = _mm_setr_pd(-1.0, 5.0); + let b = _mm_setr_pd(-100.0, 20.0); + let r = _mm_max_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(-1.0, 20.0)); + + // `_mm_max_pd` can **not** be implemented using the `simd_max` rust intrinsic because + // the semantics of `simd_max` are different to those of `_mm_max_pd` regarding handling + // of `-0.0`. + let a = _mm_setr_pd(-0.0, 0.0); + let b = _mm_setr_pd(0.0, 0.0); + let r1: [u8; 16] = transmute(_mm_max_pd(a, b)); + let r2: [u8; 16] = transmute(_mm_max_pd(b, a)); + let a: [u8; 16] = transmute(a); + let b: [u8; 16] = transmute(b); + assert_eq!(r1, b); + assert_eq!(r2, a); + assert_ne!(a, b); // sanity check that -0.0 is actually present + } + test_mm_max_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sqrt_sd() { + let a = _mm_setr_pd(1.0, 2.0); + let b = _mm_setr_pd(5.0, 10.0); + let r = _mm_sqrt_sd(a, b); + assert_eq_m128d(r, _mm_setr_pd(5.0f64.sqrt(), 2.0)); + } + test_mm_sqrt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_sqrt_pd() { + let r = _mm_sqrt_pd(_mm_setr_pd(1.0, 2.0)); + assert_eq_m128d(r, _mm_setr_pd(1.0f64.sqrt(), 2.0f64.sqrt())); + } + test_mm_sqrt_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpeq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpeq_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpeq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmplt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmplt_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmplt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmple_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmple_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmple_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpgt_sd() { + let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpgt_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpgt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpge_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpge_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpge_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpord_sd() { + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpord_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpord_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpunord_sd() { + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpunord_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpunord_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpneq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(!0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpneq_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpneq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnlt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpnlt_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnlt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnle_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpnle_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnle_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpngt_sd() { + let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpngt_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpngt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnge_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, transmute(2.0f64)); + let r = transmute::<_, __m128i>(_mm_cmpnge_sd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnge_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpeq_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, 0); + let r = transmute::<_, __m128i>(_mm_cmpeq_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpeq_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmplt_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, !0); + let r = transmute::<_, __m128i>(_mm_cmplt_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmplt_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmple_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, !0); + let r = transmute::<_, __m128i>(_mm_cmple_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmple_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpgt_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, 0); + let r = transmute::<_, __m128i>(_mm_cmpgt_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpgt_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpge_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(!0, 0); + let r = transmute::<_, __m128i>(_mm_cmpge_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpge_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpord_pd() { + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(0, !0); + let r = transmute::<_, __m128i>(_mm_cmpord_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpord_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpunord_pd() { + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(!0, 0); + let r = transmute::<_, __m128i>(_mm_cmpunord_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpunord_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpneq_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(!0, !0); + let r = transmute::<_, __m128i>(_mm_cmpneq_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpneq_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnlt_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(5.0, 3.0)); + let e = _mm_setr_epi64x(0, 0); + let r = transmute::<_, __m128i>(_mm_cmpnlt_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnlt_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnle_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, 0); + let r = transmute::<_, __m128i>(_mm_cmpnle_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnle_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpngt_pd() { + let (a, b) = (_mm_setr_pd(5.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, !0); + let r = transmute::<_, __m128i>(_mm_cmpngt_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpngt_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cmpnge_pd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + let e = _mm_setr_epi64x(0, !0); + let r = transmute::<_, __m128i>(_mm_cmpnge_pd(a, b)); + assert_eq_m128i(r, e); + } + test_mm_cmpnge_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comieq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comieq_sd(a, b) != 0); + + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comieq_sd(a, b) == 0); + } + test_mm_comieq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comilt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comilt_sd(a, b) == 0); + } + test_mm_comilt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comile_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comile_sd(a, b) != 0); + } + test_mm_comile_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comigt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comigt_sd(a, b) == 0); + } + test_mm_comigt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comige_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comige_sd(a, b) != 0); + } + test_mm_comige_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_comineq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_comineq_sd(a, b) == 0); + } + test_mm_comineq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomieq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomieq_sd(a, b) != 0); + + let (a, b) = (_mm_setr_pd(NAN, 2.0), _mm_setr_pd(NAN, 3.0)); + assert!(_mm_ucomieq_sd(a, b) == 0); + } + test_mm_ucomieq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomilt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomilt_sd(a, b) == 0); + } + test_mm_ucomilt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomile_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomile_sd(a, b) != 0); + } + test_mm_ucomile_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomigt_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomigt_sd(a, b) == 0); + } + test_mm_ucomigt_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomige_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomige_sd(a, b) != 0); + } + test_mm_ucomige_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_ucomineq_sd() { + let (a, b) = (_mm_setr_pd(1.0, 2.0), _mm_setr_pd(1.0, 3.0)); + assert!(_mm_ucomineq_sd(a, b) == 0); + } + test_mm_ucomineq_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtpd_ps() { + let r = _mm_cvtpd_ps(_mm_setr_pd(-1.0, 5.0)); + assert_eq_m128(r, _mm_setr_ps(-1.0, 5.0, 0.0, 0.0)); + + let r = _mm_cvtpd_ps(_mm_setr_pd(-1.0, -5.0)); + assert_eq_m128(r, _mm_setr_ps(-1.0, -5.0, 0.0, 0.0)); + + let r = _mm_cvtpd_ps(_mm_setr_pd(f64::MAX, f64::MIN)); + assert_eq_m128(r, _mm_setr_ps(f32::INFINITY, f32::NEG_INFINITY, 0.0, 0.0)); + + let r = _mm_cvtpd_ps(_mm_setr_pd(f32::MAX as f64, f32::MIN as f64)); + assert_eq_m128(r, _mm_setr_ps(f32::MAX, f32::MIN, 0.0, 0.0)); + } + test_mm_cvtpd_ps(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtps_pd() { + let r = _mm_cvtps_pd(_mm_setr_ps(-1.0, 2.0, -3.0, 5.0)); + assert_eq_m128d(r, _mm_setr_pd(-1.0, 2.0)); + + let r = _mm_cvtps_pd(_mm_setr_ps(f32::MAX, f32::INFINITY, f32::NEG_INFINITY, f32::MIN)); + assert_eq_m128d(r, _mm_setr_pd(f32::MAX as f64, f64::INFINITY)); + } + test_mm_cvtps_pd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtpd_epi32() { + let r = _mm_cvtpd_epi32(_mm_setr_pd(-1.0, 5.0)); + assert_eq_m128i(r, _mm_setr_epi32(-1, 5, 0, 0)); + + let r = _mm_cvtpd_epi32(_mm_setr_pd(-1.0, -5.0)); + assert_eq_m128i(r, _mm_setr_epi32(-1, -5, 0, 0)); + + let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::MAX, f64::MIN)); + assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); + + let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::INFINITY, f64::NEG_INFINITY)); + assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); + + let r = _mm_cvtpd_epi32(_mm_setr_pd(f64::NAN, f64::NAN)); + assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); + } + test_mm_cvtpd_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvttpd_epi32() { + let a = _mm_setr_pd(-1.1, 2.2); + let r = _mm_cvttpd_epi32(a); + assert_eq_m128i(r, _mm_setr_epi32(-1, 2, 0, 0)); + + let a = _mm_setr_pd(f64::NEG_INFINITY, f64::NAN); + let r = _mm_cvttpd_epi32(a); + assert_eq_m128i(r, _mm_setr_epi32(i32::MIN, i32::MIN, 0, 0)); + } + test_mm_cvttpd_epi32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtsd_si32() { + let r = _mm_cvtsd_si32(_mm_setr_pd(-2.0, 5.0)); + assert_eq!(r, -2); + + let r = _mm_cvtsd_si32(_mm_setr_pd(f64::MAX, f64::MIN)); + assert_eq!(r, i32::MIN); + + let r = _mm_cvtsd_si32(_mm_setr_pd(f64::NAN, f64::NAN)); + assert_eq!(r, i32::MIN); + } + test_mm_cvtsd_si32(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvttsd_si32() { + let a = _mm_setr_pd(-1.1, 2.2); + let r = _mm_cvttsd_si32(a); + assert_eq!(r, -1); + + let a = _mm_setr_pd(f64::NEG_INFINITY, f64::NAN); + let r = _mm_cvttsd_si32(a); + assert_eq!(r, i32::MIN); + } + test_mm_cvttsd_si32(); + + // Intrinsic only available on x86_64 + #[cfg(target_arch = "x86_64")] + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtsd_si64() { + let r = _mm_cvtsd_si64(_mm_setr_pd(-2.0, 5.0)); + assert_eq!(r, -2_i64); + + let r = _mm_cvtsd_si64(_mm_setr_pd(f64::MAX, f64::MIN)); + assert_eq!(r, i64::MIN); + } + #[cfg(target_arch = "x86_64")] + test_mm_cvtsd_si64(); + + // Intrinsic only available on x86_64 + #[cfg(target_arch = "x86_64")] + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvttsd_si64() { + let a = _mm_setr_pd(-1.1, 2.2); + let r = _mm_cvttsd_si64(a); + assert_eq!(r, -1_i64); + } + #[cfg(target_arch = "x86_64")] + test_mm_cvttsd_si64(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtsd_ss() { + let a = _mm_setr_ps(-1.1, -2.2, 3.3, 4.4); + let b = _mm_setr_pd(2.0, -5.0); + + let r = _mm_cvtsd_ss(a, b); + + assert_eq_m128(r, _mm_setr_ps(2.0, -2.2, 3.3, 4.4)); + + let a = _mm_setr_ps(-1.1, f32::NEG_INFINITY, f32::MAX, f32::NEG_INFINITY); + let b = _mm_setr_pd(f64::INFINITY, -5.0); + + let r = _mm_cvtsd_ss(a, b); + + assert_eq_m128( + r, + _mm_setr_ps(f32::INFINITY, f32::NEG_INFINITY, f32::MAX, f32::NEG_INFINITY), + ); + } + test_mm_cvtsd_ss(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_cvtss_sd() { + let a = _mm_setr_pd(-1.1, 2.2); + let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + + let r = _mm_cvtss_sd(a, b); + assert_eq_m128d(r, _mm_setr_pd(1.0, 2.2)); + + let a = _mm_setr_pd(-1.1, f64::INFINITY); + let b = _mm_setr_ps(f32::NEG_INFINITY, 2.0, 3.0, 4.0); + + let r = _mm_cvtss_sd(a, b); + assert_eq_m128d(r, _mm_setr_pd(f64::NEG_INFINITY, f64::INFINITY)); + } + test_mm_cvtss_sd(); + + #[target_feature(enable = "sse2")] + unsafe fn test_mm_movemask_pd() { + let r = _mm_movemask_pd(_mm_setr_pd(-1.0, 5.0)); + assert_eq!(r, 0b01); + + let r = _mm_movemask_pd(_mm_setr_pd(-1.0, -5.0)); + assert_eq!(r, 0b11); + } + test_mm_movemask_pd(); + } +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs new file mode 100644 index 00000000000..7566be4431b --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse3-ssse3.rs @@ -0,0 +1,395 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +// SSSE3 implicitly enables SSE3 +//@compile-flags: -C target-feature=+ssse3 + +use core::mem::transmute; +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; + +fn main() { + // SSSE3 implicitly enables SSE3, still check it to be sure + assert!(is_x86_feature_detected!("sse3")); + assert!(is_x86_feature_detected!("ssse3")); + + unsafe { + test_sse3(); + test_ssse3(); + } +} + +#[target_feature(enable = "sse3")] +unsafe fn test_sse3() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/sse3.rs + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_addsub_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_addsub_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(99.0, 25.0, 0.0, -15.0)); + } + test_mm_addsub_ps(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_addsub_pd() { + let a = _mm_setr_pd(-1.0, 5.0); + let b = _mm_setr_pd(-100.0, 20.0); + let r = _mm_addsub_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(99.0, 25.0)); + } + test_mm_addsub_pd(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_hadd_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_hadd_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(4.0, -10.0, -80.0, -5.0)); + } + test_mm_hadd_ps(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_hadd_pd() { + let a = _mm_setr_pd(-1.0, 5.0); + let b = _mm_setr_pd(-100.0, 20.0); + let r = _mm_hadd_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(4.0, -80.0)); + } + test_mm_hadd_pd(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_hsub_ps() { + let a = _mm_setr_ps(-1.0, 5.0, 0.0, -10.0); + let b = _mm_setr_ps(-100.0, 20.0, 0.0, -5.0); + let r = _mm_hsub_ps(a, b); + assert_eq_m128(r, _mm_setr_ps(-6.0, 10.0, -120.0, 5.0)); + } + test_mm_hsub_ps(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_hsub_pd() { + let a = _mm_setr_pd(-1.0, 5.0); + let b = _mm_setr_pd(-100.0, 20.0); + let r = _mm_hsub_pd(a, b); + assert_eq_m128d(r, _mm_setr_pd(-6.0, -120.0)); + } + test_mm_hsub_pd(); + + #[target_feature(enable = "sse3")] + unsafe fn test_mm_lddqu_si128() { + let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let r = _mm_lddqu_si128(&a); + assert_eq_m128i(a, r); + } + test_mm_lddqu_si128(); +} + +#[target_feature(enable = "ssse3")] +unsafe fn test_ssse3() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/ssse3.rs + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_abs_epi8() { + let r = _mm_abs_epi8(_mm_set1_epi8(-5)); + assert_eq_m128i(r, _mm_set1_epi8(5)); + } + test_mm_abs_epi8(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_abs_epi16() { + let r = _mm_abs_epi16(_mm_set1_epi16(-5)); + assert_eq_m128i(r, _mm_set1_epi16(5)); + } + test_mm_abs_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_abs_epi32() { + let r = _mm_abs_epi32(_mm_set1_epi32(-5)); + assert_eq_m128i(r, _mm_set1_epi32(5)); + } + test_mm_abs_epi32(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_shuffle_epi8() { + let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_setr_epi8(4, 128_u8 as i8, 4, 3, 24, 12, 6, 19, 12, 5, 5, 10, 4, 1, 8, 0); + let expected = _mm_setr_epi8(5, 0, 5, 4, 9, 13, 7, 4, 13, 6, 6, 11, 5, 2, 9, 1); + let r = _mm_shuffle_epi8(a, b); + assert_eq_m128i(r, expected); + + // Test indices greater than 15 wrapping around + let b = _mm_add_epi8(b, _mm_set1_epi8(32)); + let r = _mm_shuffle_epi8(a, b); + assert_eq_m128i(r, expected); + } + test_mm_shuffle_epi8(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hadd_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(4, 128, 4, 3, 24, 12, 6, 19); + let expected = _mm_setr_epi16(3, 7, 11, 15, 132, 7, 36, 25); + let r = _mm_hadd_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test wrapping on overflow + let a = _mm_setr_epi16(i16::MAX, 1, i16::MAX, 2, i16::MAX, 3, i16::MAX, 4); + let b = _mm_setr_epi16(i16::MIN, -1, i16::MIN, -2, i16::MIN, -3, i16::MIN, -4); + let expected = _mm_setr_epi16( + i16::MIN, + i16::MIN + 1, + i16::MIN + 2, + i16::MIN + 3, + i16::MAX, + i16::MAX - 1, + i16::MAX - 2, + i16::MAX - 3, + ); + let r = _mm_hadd_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hadd_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hadds_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(4, 128, 4, 3, 32767, 1, -32768, -1); + let expected = _mm_setr_epi16(3, 7, 11, 15, 132, 7, 32767, -32768); + let r = _mm_hadds_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test saturating on overflow + let a = _mm_setr_epi16(i16::MAX, 1, i16::MAX, 2, i16::MAX, 3, i16::MAX, 4); + let b = _mm_setr_epi16(i16::MIN, -1, i16::MIN, -2, i16::MIN, -3, i16::MIN, -4); + let expected = _mm_setr_epi16( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let r = _mm_hadds_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hadds_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hadd_epi32() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let b = _mm_setr_epi32(4, 128, 4, 3); + let expected = _mm_setr_epi32(3, 7, 132, 7); + let r = _mm_hadd_epi32(a, b); + assert_eq_m128i(r, expected); + + // Test wrapping on overflow + let a = _mm_setr_epi32(i32::MAX, 1, i32::MAX, 2); + let b = _mm_setr_epi32(i32::MIN, -1, i32::MIN, -2); + let expected = _mm_setr_epi32(i32::MIN, i32::MIN + 1, i32::MAX, i32::MAX - 1); + let r = _mm_hadd_epi32(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hadd_epi32(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hsub_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(4, 128, 4, 3, 24, 12, 6, 19); + let expected = _mm_setr_epi16(-1, -1, -1, -1, -124, 1, 12, -13); + let r = _mm_hsub_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test wrapping on overflow + let a = _mm_setr_epi16(i16::MAX, -1, i16::MAX, -2, i16::MAX, -3, i16::MAX, -4); + let b = _mm_setr_epi16(i16::MIN, 1, i16::MIN, 2, i16::MIN, 3, i16::MIN, 4); + let expected = _mm_setr_epi16( + i16::MIN, + i16::MIN + 1, + i16::MIN + 2, + i16::MIN + 3, + i16::MAX, + i16::MAX - 1, + i16::MAX - 2, + i16::MAX - 3, + ); + let r = _mm_hsub_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hsub_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hsubs_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(4, 128, 4, 3, 32767, -1, -32768, 1); + let expected = _mm_setr_epi16(-1, -1, -1, -1, -124, 1, 32767, -32768); + let r = _mm_hsubs_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test saturating on overflow + let a = _mm_setr_epi16(i16::MAX, -1, i16::MAX, -2, i16::MAX, -3, i16::MAX, -4); + let b = _mm_setr_epi16(i16::MIN, 1, i16::MIN, 2, i16::MIN, 3, i16::MIN, 4); + let expected = _mm_setr_epi16( + i16::MAX, + i16::MAX, + i16::MAX, + i16::MAX, + i16::MIN, + i16::MIN, + i16::MIN, + i16::MIN, + ); + let r = _mm_hsubs_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hsubs_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_hsub_epi32() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let b = _mm_setr_epi32(4, 128, 4, 3); + let expected = _mm_setr_epi32(-1, -1, -124, 1); + let r = _mm_hsub_epi32(a, b); + assert_eq_m128i(r, expected); + + // Test wrapping on overflow + let a = _mm_setr_epi32(i32::MAX, -1, i32::MAX, -2); + let b = _mm_setr_epi32(i32::MIN, 1, i32::MIN, 2); + let expected = _mm_setr_epi32(i32::MIN, i32::MIN + 1, i32::MAX, i32::MAX - 1); + let r = _mm_hsub_epi32(a, b); + assert_eq_m128i(r, expected); + } + test_mm_hsub_epi32(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_maddubs_epi16() { + let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + let b = _mm_setr_epi8(4, 63, 4, 3, 24, 12, 6, 19, 12, 5, 5, 10, 4, 1, 8, 0); + let expected = _mm_setr_epi16(130, 24, 192, 194, 158, 175, 66, 120); + let r = _mm_maddubs_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test widening and saturation + let a = _mm_setr_epi8( + u8::MAX as i8, + u8::MAX as i8, + u8::MAX as i8, + u8::MAX as i8, + u8::MAX as i8, + u8::MAX as i8, + 100, + 100, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ); + let b = _mm_setr_epi8( + i8::MAX, + i8::MAX, + i8::MAX, + i8::MIN, + i8::MIN, + i8::MIN, + 50, + 15, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ); + let expected = _mm_setr_epi16(i16::MAX, -255, i16::MIN, 6500, 0, 0, 0, 0); + let r = _mm_maddubs_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_maddubs_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_mulhrs_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8); + let b = _mm_setr_epi16(4, 128, 4, 3, 32767, -1, -32768, 1); + let expected = _mm_setr_epi16(0, 0, 0, 0, 5, 0, -7, 0); + let r = _mm_mulhrs_epi16(a, b); + assert_eq_m128i(r, expected); + + // Test extreme values + let a = _mm_setr_epi16(i16::MAX, i16::MIN, i16::MIN, 0, 0, 0, 0, 0); + let b = _mm_setr_epi16(i16::MAX, i16::MIN, i16::MAX, 0, 0, 0, 0, 0); + let expected = _mm_setr_epi16(i16::MAX - 1, i16::MIN, -i16::MAX, 0, 0, 0, 0, 0); + let r = _mm_mulhrs_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_mulhrs_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_sign_epi8() { + let a = _mm_setr_epi8(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, -14, -15, 16); + let b = _mm_setr_epi8(4, 63, -4, 3, 24, 12, -6, -19, 12, 5, -5, 10, 4, 1, -8, 0); + let expected = _mm_setr_epi8(1, 2, -3, 4, 5, 6, -7, -8, 9, 10, -11, 12, 13, -14, 15, 0); + let r = _mm_sign_epi8(a, b); + assert_eq_m128i(r, expected); + } + test_mm_sign_epi8(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_sign_epi16() { + let a = _mm_setr_epi16(1, 2, 3, 4, -5, -6, 7, 8); + let b = _mm_setr_epi16(4, 128, 0, 3, 1, -1, -2, 1); + let expected = _mm_setr_epi16(1, 2, 0, 4, -5, 6, -7, 8); + let r = _mm_sign_epi16(a, b); + assert_eq_m128i(r, expected); + } + test_mm_sign_epi16(); + + #[target_feature(enable = "ssse3")] + unsafe fn test_mm_sign_epi32() { + let a = _mm_setr_epi32(-1, 2, 3, 4); + let b = _mm_setr_epi32(1, -1, 1, 0); + let expected = _mm_setr_epi32(-1, -2, 3, 0); + let r = _mm_sign_epi32(a, b); + assert_eq_m128i(r, expected); + } + test_mm_sign_epi32(); +} + +#[track_caller] +#[target_feature(enable = "sse")] +unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +pub unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs new file mode 100644 index 00000000000..06607f3fd59 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86-sse41.rs @@ -0,0 +1,548 @@ +// Ignore everything except x86 and x86_64 +// Any new targets that are added to CI should be ignored here. +// (We cannot use `cfg`-based tricks here since the `target-feature` flags below only work on x86.) +//@ignore-target-aarch64 +//@ignore-target-arm +//@ignore-target-avr +//@ignore-target-s390x +//@ignore-target-thumbv7em +//@ignore-target-wasm32 +//@compile-flags: -C target-feature=+sse4.1 + +#[cfg(target_arch = "x86")] +use std::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use std::arch::x86_64::*; +use std::mem::transmute; + +fn main() { + assert!(is_x86_feature_detected!("sse4.1")); + + unsafe { + test_sse41(); + } +} + +#[target_feature(enable = "sse4.1")] +unsafe fn test_sse41() { + // Mostly copied from library/stdarch/crates/core_arch/src/x86/sse41.rs + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_insert_ps() { + let a = _mm_set1_ps(1.0); + let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let r = _mm_insert_ps::<0b11_00_1100>(a, b); + let e = _mm_setr_ps(4.0, 1.0, 0.0, 0.0); + assert_eq_m128(r, e); + + // Zeroing takes precedence over copied value + let a = _mm_set1_ps(1.0); + let b = _mm_setr_ps(1.0, 2.0, 3.0, 4.0); + let r = _mm_insert_ps::<0b11_00_0001>(a, b); + let e = _mm_setr_ps(0.0, 1.0, 1.0, 1.0); + assert_eq_m128(r, e); + } + test_mm_insert_ps(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_packus_epi32() { + let a = _mm_setr_epi32(1, 2, 3, 4); + let b = _mm_setr_epi32(-1, -2, -3, -4); + let r = _mm_packus_epi32(a, b); + let e = _mm_setr_epi16(1, 2, 3, 4, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + test_mm_packus_epi32(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_dp_pd() { + let a = _mm_setr_pd(2.0, 3.0); + let b = _mm_setr_pd(1.0, 4.0); + let e = _mm_setr_pd(14.0, 0.0); + assert_eq_m128d(_mm_dp_pd::<0b00110001>(a, b), e); + } + test_mm_dp_pd(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_dp_ps() { + let a = _mm_setr_ps(2.0, 3.0, 1.0, 10.0); + let b = _mm_setr_ps(1.0, 4.0, 0.5, 10.0); + let e = _mm_setr_ps(14.5, 0.0, 14.5, 0.0); + assert_eq_m128(_mm_dp_ps::<0b01110101>(a, b), e); + } + test_mm_dp_ps(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_nearest_f32() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f32, res: f32) { + let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); + let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); + let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); + let r = _mm_round_ss::<_MM_FROUND_TO_NEAREST_INT>(a, b); + assert_eq_m128(r, e); + // Assume round-to-nearest by default + let r = _mm_round_ss::<_MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq_m128(r, e); + + let a = _mm_set1_ps(x); + let e = _mm_set1_ps(res); + let r = _mm_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m128(r, e); + // Assume round-to-nearest by default + let r = _mm_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m128(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); + let e = _mm_setr_ps(2.0, 4.0, 6.0, 8.0); + let r = _mm_round_ps::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m128(r, e); + // Assume round-to-nearest by default + let r = _mm_round_ps::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m128(r, e); + } + test_round_nearest_f32(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_floor_f32() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f32, res: f32) { + let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); + let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); + let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); + let r = _mm_floor_ss(a, b); + assert_eq_m128(r, e); + let r = _mm_round_ss::<_MM_FROUND_TO_NEG_INF>(a, b); + assert_eq_m128(r, e); + + let a = _mm_set1_ps(x); + let e = _mm_set1_ps(res); + let r = _mm_floor_ps(a); + assert_eq_m128(r, e); + let r = _mm_round_ps::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m128(r, e); + } + + // Test rounding direction + test(-2.5, -3.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -2.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); + let e = _mm_setr_ps(1.0, 3.0, 5.0, 7.0); + let r = _mm_floor_ps(a); + assert_eq_m128(r, e); + let r = _mm_round_ps::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m128(r, e); + } + test_round_floor_f32(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_ceil_f32() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f32, res: f32) { + let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); + let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); + let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); + let r = _mm_ceil_ss(a, b); + assert_eq_m128(r, e); + let r = _mm_round_ss::<_MM_FROUND_TO_POS_INF>(a, b); + assert_eq_m128(r, e); + + let a = _mm_set1_ps(x); + let e = _mm_set1_ps(res); + let r = _mm_ceil_ps(a); + assert_eq_m128(r, e); + let r = _mm_round_ps::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m128(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 2.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 3.0); + + // Test that each element is rounded + let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); + let e = _mm_setr_ps(2.0, 4.0, 6.0, 8.0); + let r = _mm_ceil_ps(a); + assert_eq_m128(r, e); + let r = _mm_round_ps::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m128(r, e); + } + test_round_ceil_f32(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_trunc_f32() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f32, res: f32) { + let a = _mm_setr_ps(3.5, 2.5, 1.5, 4.5); + let b = _mm_setr_ps(x, -1.5, -3.5, -2.5); + let e = _mm_setr_ps(res, 2.5, 1.5, 4.5); + let r = _mm_round_ss::<_MM_FROUND_TO_ZERO>(a, b); + assert_eq_m128(r, e); + + let a = _mm_set1_ps(x); + let e = _mm_set1_ps(res); + let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m128(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_ps(1.5, 3.5, 5.5, 7.5); + let e = _mm_setr_ps(1.0, 3.0, 5.0, 7.0); + let r = _mm_round_ps::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m128(r, e); + } + test_round_trunc_f32(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_nearest_f64() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f64, res: f64) { + let a = _mm_setr_pd(3.5, 2.5); + let b = _mm_setr_pd(x, -1.5); + let e = _mm_setr_pd(res, 2.5); + let r = _mm_round_sd::<_MM_FROUND_TO_NEAREST_INT>(a, b); + assert_eq_m128d(r, e); + // Assume round-to-nearest by default + let r = _mm_round_sd::<_MM_FROUND_CUR_DIRECTION>(a, b); + assert_eq_m128d(r, e); + + let a = _mm_set1_pd(x); + let e = _mm_set1_pd(res); + let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m128d(r, e); + // Assume round-to-nearest by default + let r = _mm_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m128d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_pd(1.5, 3.5); + let e = _mm_setr_pd(2.0, 4.0); + let r = _mm_round_pd::<_MM_FROUND_TO_NEAREST_INT>(a); + assert_eq_m128d(r, e); + // Assume round-to-nearest by default + let r = _mm_round_pd::<_MM_FROUND_CUR_DIRECTION>(a); + assert_eq_m128d(r, e); + } + test_round_nearest_f64(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_floor_f64() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f64, res: f64) { + let a = _mm_setr_pd(3.5, 2.5); + let b = _mm_setr_pd(x, -1.5); + let e = _mm_setr_pd(res, 2.5); + let r = _mm_floor_sd(a, b); + assert_eq_m128d(r, e); + let r = _mm_round_sd::<_MM_FROUND_TO_NEG_INF>(a, b); + assert_eq_m128d(r, e); + + let a = _mm_set1_pd(x); + let e = _mm_set1_pd(res); + let r = _mm_floor_pd(a); + assert_eq_m128d(r, e); + let r = _mm_round_pd::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m128d(r, e); + } + + // Test rounding direction + test(-2.5, -3.0); + test(-1.75, -2.0); + test(-1.5, -2.0); + test(-1.25, -2.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_pd(1.5, 3.5); + let e = _mm_setr_pd(1.0, 3.0); + let r = _mm_floor_pd(a); + assert_eq_m128d(r, e); + let r = _mm_round_pd::<_MM_FROUND_TO_NEG_INF>(a); + assert_eq_m128d(r, e); + } + test_round_floor_f64(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_ceil_f64() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f64, res: f64) { + let a = _mm_setr_pd(3.5, 2.5); + let b = _mm_setr_pd(x, -1.5); + let e = _mm_setr_pd(res, 2.5); + let r = _mm_ceil_sd(a, b); + assert_eq_m128d(r, e); + let r = _mm_round_sd::<_MM_FROUND_TO_POS_INF>(a, b); + assert_eq_m128d(r, e); + + let a = _mm_set1_pd(x); + let e = _mm_set1_pd(res); + let r = _mm_ceil_pd(a); + assert_eq_m128d(r, e); + let r = _mm_round_pd::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m128d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 2.0); + test(1.5, 2.0); + test(1.75, 2.0); + test(2.5, 3.0); + + // Test that each element is rounded + let a = _mm_setr_pd(1.5, 3.5); + let e = _mm_setr_pd(2.0, 4.0); + let r = _mm_ceil_pd(a); + assert_eq_m128d(r, e); + let r = _mm_round_pd::<_MM_FROUND_TO_POS_INF>(a); + assert_eq_m128d(r, e); + } + test_round_ceil_f64(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_round_trunc_f64() { + #[target_feature(enable = "sse4.1")] + unsafe fn test(x: f64, res: f64) { + let a = _mm_setr_pd(3.5, 2.5); + let b = _mm_setr_pd(x, -1.5); + let e = _mm_setr_pd(res, 2.5); + let r = _mm_round_sd::<_MM_FROUND_TO_ZERO>(a, b); + assert_eq_m128d(r, e); + + let a = _mm_set1_pd(x); + let e = _mm_set1_pd(res); + let r = _mm_round_pd::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m128d(r, e); + } + + // Test rounding direction + test(-2.5, -2.0); + test(-1.75, -1.0); + test(-1.5, -1.0); + test(-1.25, -1.0); + test(-1.0, -1.0); + test(0.0, 0.0); + test(1.0, 1.0); + test(1.25, 1.0); + test(1.5, 1.0); + test(1.75, 1.0); + test(2.5, 2.0); + + // Test that each element is rounded + let a = _mm_setr_pd(1.5, 3.5); + let e = _mm_setr_pd(1.0, 3.0); + let r = _mm_round_pd::<_MM_FROUND_TO_ZERO>(a); + assert_eq_m128d(r, e); + } + test_round_trunc_f64(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_minpos_epu16() { + let a = _mm_setr_epi16(23, 18, 44, 97, 50, 13, 67, 66); + let r = _mm_minpos_epu16(a); + let e = _mm_setr_epi16(13, 5, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + + let a = _mm_setr_epi16(0, 18, 44, 97, 50, 13, 67, 66); + let r = _mm_minpos_epu16(a); + let e = _mm_setr_epi16(0, 0, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + + // Case where the minimum value is repeated + let a = _mm_setr_epi16(23, 18, 44, 97, 50, 13, 67, 13); + let r = _mm_minpos_epu16(a); + let e = _mm_setr_epi16(13, 5, 0, 0, 0, 0, 0, 0); + assert_eq_m128i(r, e); + } + test_mm_minpos_epu16(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_mpsadbw_epu8() { + let a = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + + let r = _mm_mpsadbw_epu8::<0b000>(a, a); + let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); + assert_eq_m128i(r, e); + + let r = _mm_mpsadbw_epu8::<0b001>(a, a); + let e = _mm_setr_epi16(16, 12, 8, 4, 0, 4, 8, 12); + assert_eq_m128i(r, e); + + let r = _mm_mpsadbw_epu8::<0b100>(a, a); + let e = _mm_setr_epi16(16, 20, 24, 28, 32, 36, 40, 44); + assert_eq_m128i(r, e); + + let r = _mm_mpsadbw_epu8::<0b101>(a, a); + let e = _mm_setr_epi16(0, 4, 8, 12, 16, 20, 24, 28); + assert_eq_m128i(r, e); + + let r = _mm_mpsadbw_epu8::<0b111>(a, a); + let e = _mm_setr_epi16(32, 28, 24, 20, 16, 12, 8, 4); + assert_eq_m128i(r, e); + } + test_mm_mpsadbw_epu8(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_testz_si128() { + let a = _mm_set1_epi8(1); + let mask = _mm_set1_epi8(0); + let r = _mm_testz_si128(a, mask); + assert_eq!(r, 1); + + let a = _mm_set1_epi8(0b101); + let mask = _mm_set1_epi8(0b110); + let r = _mm_testz_si128(a, mask); + assert_eq!(r, 0); + + let a = _mm_set1_epi8(0b011); + let mask = _mm_set1_epi8(0b100); + let r = _mm_testz_si128(a, mask); + assert_eq!(r, 1); + } + test_mm_testz_si128(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_testc_si128() { + let a = _mm_set1_epi8(-1); + let mask = _mm_set1_epi8(0); + let r = _mm_testc_si128(a, mask); + assert_eq!(r, 1); + + let a = _mm_set1_epi8(0b101); + let mask = _mm_set1_epi8(0b110); + let r = _mm_testc_si128(a, mask); + assert_eq!(r, 0); + + let a = _mm_set1_epi8(0b101); + let mask = _mm_set1_epi8(0b100); + let r = _mm_testc_si128(a, mask); + assert_eq!(r, 1); + } + test_mm_testc_si128(); + + #[target_feature(enable = "sse4.1")] + unsafe fn test_mm_testnzc_si128() { + let a = _mm_set1_epi8(0); + let mask = _mm_set1_epi8(1); + let r = _mm_testnzc_si128(a, mask); + assert_eq!(r, 0); + + let a = _mm_set1_epi8(-1); + let mask = _mm_set1_epi8(0); + let r = _mm_testnzc_si128(a, mask); + assert_eq!(r, 0); + + let a = _mm_set1_epi8(0b101); + let mask = _mm_set1_epi8(0b110); + let r = _mm_testnzc_si128(a, mask); + assert_eq!(r, 1); + + let a = _mm_set1_epi8(0b101); + let mask = _mm_set1_epi8(0b101); + let r = _mm_testnzc_si128(a, mask); + assert_eq!(r, 0); + + let a = _mm_setr_epi32(0b100, 0, 0, 0b010); + let mask = _mm_setr_epi32(0b100, 0, 0, 0b110); + let r = _mm_testnzc_si128(a, mask); + assert_eq!(r, 1); + } + test_mm_testnzc_si128(); +} + +#[track_caller] +#[target_feature(enable = "sse")] +unsafe fn assert_eq_m128(a: __m128, b: __m128) { + let r = _mm_cmpeq_ps(a, b); + if _mm_movemask_ps(r) != 0b1111 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) { + if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 { + panic!("{:?} != {:?}", a, b); + } +} + +#[track_caller] +#[target_feature(enable = "sse2")] +pub unsafe fn assert_eq_m128i(a: __m128i, b: __m128i) { + assert_eq!(transmute::<_, [u64; 2]>(a), transmute::<_, [u64; 2]>(b)) +} diff --git a/src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs new file mode 100644 index 00000000000..90bcdba4353 --- /dev/null +++ b/src/tools/miri/tests/pass/shims/x86/intrinsics-x86.rs @@ -0,0 +1,99 @@ +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod x86 { + #[cfg(target_arch = "x86")] + use core::arch::x86 as arch; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64 as arch; + + fn adc(c_in: u8, a: u32, b: u32) -> (u8, u32) { + let mut sum = 0; + // SAFETY: There are no safety requirements for calling `_addcarry_u32`. + // It's just unsafe for API consistency with other intrinsics. + let c_out = unsafe { arch::_addcarry_u32(c_in, a, b, &mut sum) }; + (c_out, sum) + } + + fn sbb(b_in: u8, a: u32, b: u32) -> (u8, u32) { + let mut sum = 0; + // SAFETY: There are no safety requirements for calling `_subborrow_u32`. + // It's just unsafe for API consistency with other intrinsics. + let b_out = unsafe { arch::_subborrow_u32(b_in, a, b, &mut sum) }; + (b_out, sum) + } + + pub fn main() { + assert_eq!(adc(0, 1, 1), (0, 2)); + assert_eq!(adc(1, 1, 1), (0, 3)); + assert_eq!(adc(2, 1, 1), (0, 3)); // any non-zero carry acts as 1! + assert_eq!(adc(u8::MAX, 1, 1), (0, 3)); + assert_eq!(adc(0, u32::MAX, u32::MAX), (1, u32::MAX - 1)); + assert_eq!(adc(1, u32::MAX, u32::MAX), (1, u32::MAX)); + assert_eq!(adc(2, u32::MAX, u32::MAX), (1, u32::MAX)); + assert_eq!(adc(u8::MAX, u32::MAX, u32::MAX), (1, u32::MAX)); + + assert_eq!(sbb(0, 1, 1), (0, 0)); + assert_eq!(sbb(1, 1, 1), (1, u32::MAX)); + assert_eq!(sbb(2, 1, 1), (1, u32::MAX)); // any non-zero borrow acts as 1! + assert_eq!(sbb(u8::MAX, 1, 1), (1, u32::MAX)); + assert_eq!(sbb(0, 2, 1), (0, 1)); + assert_eq!(sbb(1, 2, 1), (0, 0)); + assert_eq!(sbb(2, 2, 1), (0, 0)); + assert_eq!(sbb(u8::MAX, 2, 1), (0, 0)); + assert_eq!(sbb(0, 1, 2), (1, u32::MAX)); + assert_eq!(sbb(1, 1, 2), (1, u32::MAX - 1)); + assert_eq!(sbb(2, 1, 2), (1, u32::MAX - 1)); + assert_eq!(sbb(u8::MAX, 1, 2), (1, u32::MAX - 1)); + } +} + +#[cfg(target_arch = "x86_64")] +mod x86_64 { + use core::arch::x86_64 as arch; + + fn adc(c_in: u8, a: u64, b: u64) -> (u8, u64) { + let mut sum = 0; + // SAFETY: There are no safety requirements for calling `_addcarry_u64`. + // It's just unsafe for API consistency with other intrinsics. + let c_out = unsafe { arch::_addcarry_u64(c_in, a, b, &mut sum) }; + (c_out, sum) + } + + fn sbb(b_in: u8, a: u64, b: u64) -> (u8, u64) { + let mut sum = 0; + // SAFETY: There are no safety requirements for calling `_subborrow_u64`. + // It's just unsafe for API consistency with other intrinsics. + let b_out = unsafe { arch::_subborrow_u64(b_in, a, b, &mut sum) }; + (b_out, sum) + } + + pub fn main() { + assert_eq!(adc(0, 1, 1), (0, 2)); + assert_eq!(adc(1, 1, 1), (0, 3)); + assert_eq!(adc(2, 1, 1), (0, 3)); // any non-zero carry acts as 1! + assert_eq!(adc(u8::MAX, 1, 1), (0, 3)); + assert_eq!(adc(0, u64::MAX, u64::MAX), (1, u64::MAX - 1)); + assert_eq!(adc(1, u64::MAX, u64::MAX), (1, u64::MAX)); + assert_eq!(adc(2, u64::MAX, u64::MAX), (1, u64::MAX)); + assert_eq!(adc(u8::MAX, u64::MAX, u64::MAX), (1, u64::MAX)); + + assert_eq!(sbb(0, 1, 1), (0, 0)); + assert_eq!(sbb(1, 1, 1), (1, u64::MAX)); + assert_eq!(sbb(2, 1, 1), (1, u64::MAX)); // any non-zero borrow acts as 1! + assert_eq!(sbb(u8::MAX, 1, 1), (1, u64::MAX)); + assert_eq!(sbb(0, 2, 1), (0, 1)); + assert_eq!(sbb(1, 2, 1), (0, 0)); + assert_eq!(sbb(2, 2, 1), (0, 0)); + assert_eq!(sbb(u8::MAX, 2, 1), (0, 0)); + assert_eq!(sbb(0, 1, 2), (1, u64::MAX)); + assert_eq!(sbb(1, 1, 2), (1, u64::MAX - 1)); + assert_eq!(sbb(2, 1, 2), (1, u64::MAX - 1)); + assert_eq!(sbb(u8::MAX, 1, 2), (1, u64::MAX - 1)); + } +} + +fn main() { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + x86::main(); + #[cfg(target_arch = "x86_64")] + x86_64::main(); +} diff --git a/src/tools/miri/tests/pass/volatile.rs b/src/tools/miri/tests/pass/volatile.rs deleted file mode 100644 index c9799801455..00000000000 --- a/src/tools/miri/tests/pass/volatile.rs +++ /dev/null @@ -1,12 +0,0 @@ -// related: #58645 -#![feature(core_intrinsics)] -use std::intrinsics::{volatile_load, volatile_store}; - -pub fn main() { - unsafe { - let i: &mut (isize, isize) = &mut (0, 0); - volatile_store(i, (1, 2)); - assert_eq!(volatile_load(i), (1, 2)); - assert_eq!(i, &mut (1, 2)); - } -} -- cgit 1.4.1-3-g733a5