Auto merge of #78863 - KodrAus:feat/simd-array, r=oli-obk

Support repr(simd) on ADTs containing a single array field

This is a squash and rebase of `@gnzlbg's` #63531

I've never actually written code in the compiler before so just fumbled my way around until it would build 😅

I imagine there'll be some work we need to do in `rustc_codegen_cranelift` too for this now, but might need some input from `@bjorn3` to know what that is.

cc `@rust-lang/project-portable-simd`

-----

This PR allows using `#[repr(simd)]` on ADTs containing a single array field:

```rust
 #[repr(simd)] struct S0([f32; 4]);
 #[repr(simd)] struct S1<const N: usize>([f32; N]);
 #[repr(simd)] struct S2<T, const N: usize>([T; N]);
```

This should allow experimenting with portable packed SIMD abstractions on nightly that make use of const generics.

13 files changed, 306 insertions, 71 deletions

diff --git a/src/test/codegen/simd-intrinsic/simd-intrinsic-generic-extract-insert.rs b/src/test/codegen/simd-intrinsic/simd-intrinsic-generic-extract-insert.rs
new file mode 100644
index 00000000000..b5b0b1330a6
--- /dev/null
+++ b/src/test/codegen/simd-intrinsic/simd-intrinsic-generic-extract-insert.rs
@@ -0,0 +1,47 @@
+// compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
+#![allow(non_camel_case_types)]
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct M(pub f32, pub f32, pub f32, pub f32);
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct S<const N: usize>([f32; N]);
+
+extern "platform-intrinsic" {
+    fn simd_extract<T, U>(x: T, idx: u32) -> U;
+    fn simd_insert<T, U>(x: T, idx: u32, b: U) -> T;
+}
+
+// CHECK-LABEL: @extract_m
+#[no_mangle]
+pub unsafe fn extract_m(v: M, i: u32) -> f32  {
+    // CHECK: extractelement <4 x float> %{{v|_3}}, i32 %i
+    simd_extract(v, i)
+}
+
+// CHECK-LABEL: @extract_s
+#[no_mangle]
+pub unsafe fn extract_s(v: S<4>, i: u32) -> f32  {
+    // CHECK: extractelement <4 x float> %{{v|_3}}, i32 %i
+    simd_extract(v, i)
+}
+
+// CHECK-LABEL: @insert_m
+#[no_mangle]
+pub unsafe fn insert_m(v: M, i: u32, j: f32) -> M  {
+    // CHECK: insertelement <4 x float> %{{v|_4}}, float %j, i32 %i
+    simd_insert(v, i, j)
+}
+
+// CHECK-LABEL: @insert_s
+#[no_mangle]
+pub unsafe fn insert_s(v: S<4>, i: u32, j: f32) -> S<4>  {
+    // CHECK: insertelement <4 x float> %{{v|_4}}, float %j, i32 %i
+    simd_insert(v, i, j)
+}
diff --git a/src/test/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs b/src/test/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs
new file mode 100644
index 00000000000..56466a78d5a
--- /dev/null
+++ b/src/test/codegen/simd-intrinsic/simd-intrinsic-transmute-array.rs
@@ -0,0 +1,44 @@
+// ignore-tidy-linelength
+// compile-flags: -C no-prepopulate-passes
+// min-llvm-version 8.0
+
+#![crate_type = "lib"]
+
+#![allow(non_camel_case_types)]
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct S<const N: usize>([f32; N]);
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct T([f32; 4]);
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct U(f32, f32, f32, f32);
+
+// CHECK-LABEL: @build_array_s
+#[no_mangle]
+pub fn build_array_s(x: [f32; 4]) -> S<4> {
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    S::<4>(x)
+}
+
+// CHECK-LABEL: @build_array_t
+#[no_mangle]
+pub fn build_array_t(x: [f32; 4]) -> T {
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    T(x)
+}
+
+// CHECK-LABEL: @build_array_u
+#[no_mangle]
+pub fn build_array_u(x: [f32; 4]) -> U {
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{[0-9]+}}(i8* {{.*}} %{{[0-9]+}}, i8* {{.*}} %{{[0-9]+}}, i{{[0-9]+}} 16, i1 false)
+    unsafe { std::mem::transmute(x) }
+}
diff --git a/src/test/ui/error-codes/E0077.stderr b/src/test/ui/error-codes/E0077.stderr
index 4f85d175ce6..1938a9a272a 100644
--- a/src/test/ui/error-codes/E0077.stderr
+++ b/src/test/ui/error-codes/E0077.stderr
@@ -1,4 +1,4 @@
-error[E0077]: SIMD vector element type should be machine type
+error[E0077]: SIMD vector element type should be a primitive scalar (integer/float/pointer) type
   --> $DIR/E0077.rs:4:1
    |
 LL | struct Bad(String);
diff --git a/src/test/ui/simd-type-generic-monomorphisation.rs b/src/test/ui/simd-type-generic-monomorphisation.rs
index fc5e23f4986..0275f0ce4c1 100644
--- a/src/test/ui/simd-type-generic-monomorphisation.rs
+++ b/src/test/ui/simd-type-generic-monomorphisation.rs
@@ -2,7 +2,9 @@
 
 #![feature(repr_simd, platform_intrinsics)]
 
-// error-pattern:monomorphising SIMD type `Simd2<X>` with a non-machine element type `X`
+// ignore-tidy-linelength
+
+// error-pattern:monomorphising SIMD type `Simd2<X>` with a non-primitive-scalar (integer/float/pointer) element type `X`
 
 struct X(Vec<i32>);
 #[repr(simd)]
diff --git a/src/test/ui/simd-type-generic-monomorphisation.stderr b/src/test/ui/simd-type-generic-monomorphisation.stderr
index 2a74506e80e..7f23893ac85 100644
--- a/src/test/ui/simd-type-generic-monomorphisation.stderr
+++ b/src/test/ui/simd-type-generic-monomorphisation.stderr
@@ -1,4 +1,4 @@
-error: monomorphising SIMD type `Simd2<X>` with a non-machine element type `X`
+error: monomorphising SIMD type `Simd2<X>` with a non-primitive-scalar (integer/float/pointer) element type `X`
 
 error: aborting due to previous error
 
diff --git a/src/test/ui/simd-type.rs b/src/test/ui/simd-type.rs
index 9e4b7e76560..a320df85138 100644
--- a/src/test/ui/simd-type.rs
+++ b/src/test/ui/simd-type.rs
@@ -1,10 +1,20 @@
 #![feature(repr_simd)]
 #![allow(non_camel_case_types)]
 
+// ignore-tidy-linelength
+
 #[repr(simd)]
 struct empty; //~ ERROR SIMD vector cannot be empty
 
 #[repr(simd)]
 struct i64f64(i64, f64); //~ ERROR SIMD vector should be homogeneous
 
+struct Foo;
+
+#[repr(simd)]
+struct FooV(Foo, Foo); //~ ERROR SIMD vector element type should be a primitive scalar (integer/float/pointer) type
+
+#[repr(simd)]
+struct FooV2([Foo; 2]); //~ ERROR SIMD vector element type should be a primitive scalar (integer/float/pointer) type
+
 fn main() {}
diff --git a/src/test/ui/simd-type.stderr b/src/test/ui/simd-type.stderr
index 0c4242f46b7..23004c78591 100644
--- a/src/test/ui/simd-type.stderr
+++ b/src/test/ui/simd-type.stderr
@@ -1,16 +1,28 @@
 error[E0075]: SIMD vector cannot be empty
-  --> $DIR/simd-type.rs:5:1
+  --> $DIR/simd-type.rs:7:1
    |
 LL | struct empty;
    | ^^^^^^^^^^^^^
 
 error[E0076]: SIMD vector should be homogeneous
-  --> $DIR/simd-type.rs:8:1
+  --> $DIR/simd-type.rs:10:1
    |
 LL | struct i64f64(i64, f64);
    | ^^^^^^^^^^^^^^^^^^^^^^^^ SIMD elements must have the same type
 
-error: aborting due to 2 previous errors
+error[E0077]: SIMD vector element type should be a primitive scalar (integer/float/pointer) type
+  --> $DIR/simd-type.rs:15:1
+   |
+LL | struct FooV(Foo, Foo);
+   | ^^^^^^^^^^^^^^^^^^^^^^
+
+error[E0077]: SIMD vector element type should be a primitive scalar (integer/float/pointer) type
+  --> $DIR/simd-type.rs:18:1
+   |
+LL | struct FooV2([Foo; 2]);
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 4 previous errors
 
-Some errors have detailed explanations: E0075, E0076.
+Some errors have detailed explanations: E0075, E0076, E0077.
 For more information about an error, try `rustc --explain E0075`.
diff --git a/src/test/ui/simd/simd-array-trait.rs b/src/test/ui/simd/simd-array-trait.rs
new file mode 100644
index 00000000000..d6ed5f61a00
--- /dev/null
+++ b/src/test/ui/simd/simd-array-trait.rs
@@ -0,0 +1,41 @@
+// Figuring out the size of a vector type that depends on traits doesn't ICE
+
+#![allow(dead_code)]
+
+// pretty-expanded FIXME #23616
+
+#![feature(repr_simd, platform_intrinsics, const_generics)]
+#![allow(non_camel_case_types, incomplete_features)]
+
+pub trait Simd {
+    type Lane: Clone + Copy;
+    const SIZE: usize;
+}
+
+pub struct i32x4;
+impl Simd for i32x4 {
+    type Lane = i32;
+    const SIZE: usize = 4;
+}
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+pub struct T<S: Simd>([S::Lane; S::SIZE]);
+//~^ ERROR constant expression depends on a generic parameter
+
+extern "platform-intrinsic" {
+    fn simd_insert<T, E>(x: T, idx: u32, y: E) -> T;
+    fn simd_extract<T, E>(x: T, idx: u32) -> E;
+}
+
+pub fn main() {
+    let mut t = T::<i32x4>([0; 4]);
+    unsafe {
+        for i in 0_i32..4 {
+            t = simd_insert(t, i as u32, i);
+        }
+        for i in 0_i32..4 {
+            assert_eq!(i, simd_extract(t, i as u32));
+        }
+    }
+}
diff --git a/src/test/ui/simd/simd-array-trait.stderr b/src/test/ui/simd/simd-array-trait.stderr
new file mode 100644
index 00000000000..c100e020c54
--- /dev/null
+++ b/src/test/ui/simd/simd-array-trait.stderr
@@ -0,0 +1,10 @@
+error: constant expression depends on a generic parameter
+  --> $DIR/simd-array-trait.rs:23:23
+   |
+LL | pub struct T<S: Simd>([S::Lane; S::SIZE]);
+   |                       ^^^^^^^^^^^^^^^^^^
+   |
+   = note: this may fail depending on what value the parameter takes
+
+error: aborting due to previous error
+
diff --git a/src/test/ui/simd/simd-array-type.rs b/src/test/ui/simd/simd-array-type.rs
new file mode 100644
index 00000000000..e84186a42ad
--- /dev/null
+++ b/src/test/ui/simd/simd-array-type.rs
@@ -0,0 +1,42 @@
+// run-pass
+#![allow(dead_code)]
+
+// pretty-expanded FIXME #23616
+
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+struct S([i32; 4]);
+
+#[repr(simd)]
+#[derive(Copy, Clone)]
+struct T<const N: usize>([i32; N]);
+
+extern "platform-intrinsic" {
+    fn simd_insert<T, E>(x: T, idx: u32, y: E) -> T;
+    fn simd_extract<T, E>(x: T, idx: u32) -> E;
+}
+
+pub fn main() {
+    let mut s = S([0; 4]);
+
+    unsafe {
+        for i in 0_i32..4 {
+            s = simd_insert(s, i as u32, i);
+        }
+        for i in 0_i32..4 {
+            assert_eq!(i, simd_extract(s, i as u32));
+        }
+    }
+
+    let mut t = T::<4>([0; 4]);
+    unsafe {
+        for i in 0_i32..4 {
+            t = simd_insert(t, i as u32, i);
+        }
+        for i in 0_i32..4 {
+            assert_eq!(i, simd_extract(t, i as u32));
+        }
+    }
+}
diff --git a/src/test/ui/simd/simd-generics.rs b/src/test/ui/simd/simd-generics.rs
index ab6caee9d7b..dedca6276cd 100644
--- a/src/test/ui/simd/simd-generics.rs
+++ b/src/test/ui/simd/simd-generics.rs
@@ -1,9 +1,6 @@
 // run-pass
 #![allow(non_camel_case_types)]
-
-
-
-#![feature(repr_simd, platform_intrinsics)]
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
 
 use std::ops;
 
@@ -11,6 +8,11 @@ use std::ops;
 #[derive(Copy, Clone)]
 struct f32x4(f32, f32, f32, f32);
 
+#[repr(simd)]
+#[derive(Copy, Clone)]
+struct S<const N: usize>([f32; N]);
+
+
 extern "platform-intrinsic" {
     fn simd_add<T>(x: T, y: T) -> T;
 }
@@ -23,10 +25,19 @@ impl ops::Add for f32x4 {
     type Output = f32x4;
 
     fn add(self, rhs: f32x4) -> f32x4 {
-        unsafe {simd_add(self, rhs)}
+        unsafe { simd_add(self, rhs) }
     }
 }
 
+impl ops::Add for S<4> {
+    type Output = Self;
+
+    fn add(self, rhs: Self) -> Self {
+        unsafe { simd_add(self, rhs) }
+    }
+}
+
+
 pub fn main() {
     let lr = f32x4(1.0f32, 2.0f32, 3.0f32, 4.0f32);
 
@@ -36,4 +47,11 @@ pub fn main() {
     assert_eq!(y, 4.0f32);
     assert_eq!(z, 6.0f32);
     assert_eq!(w, 8.0f32);
+
+    let lr2 = S::<4>([1.0f32, 2.0f32, 3.0f32, 4.0f32]);
+    let [x, y, z, w] = add(lr2, lr2).0;
+    assert_eq!(x, 2.0f32);
+    assert_eq!(y, 4.0f32);
+    assert_eq!(z, 6.0f32);
+    assert_eq!(w, 8.0f32);
 }
diff --git a/src/test/ui/simd/simd-intrinsic-generic-arithmetic-saturating.rs b/src/test/ui/simd/simd-intrinsic-generic-arithmetic-saturating.rs
index b7b3ec99781..4c459fa4bc8 100644
--- a/src/test/ui/simd/simd-intrinsic-generic-arithmetic-saturating.rs
+++ b/src/test/ui/simd/simd-intrinsic-generic-arithmetic-saturating.rs
@@ -2,15 +2,15 @@
 // ignore-emscripten
 
 #![allow(non_camel_case_types)]
-#![feature(repr_simd, platform_intrinsics)]
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
 
 #[repr(simd)]
 #[derive(Copy, Clone, PartialEq, Debug)]
 struct u32x4(pub u32, pub u32, pub u32, pub u32);
 
 #[repr(simd)]
-#[derive(Copy, Clone, PartialEq, Debug)]
-struct i32x4(pub i32, pub i32, pub i32, pub i32);
+#[derive(Copy, Clone)]
+struct I32<const N: usize>([i32; N]);
 
 extern "platform-intrinsic" {
     fn simd_saturating_add<T>(x: T, y: T) -> T;
@@ -51,41 +51,41 @@ fn main() {
         const MIN: i32 = i32::MIN;
         const MAX: i32 = i32::MAX;
 
-        let a = i32x4(1, 2, 3, 4);
-        let b = i32x4(2, 4, 6, 8);
-        let c = i32x4(-1, -2, -3, -4);
-        let d = i32x4(-2, -4, -6, -8);
+        let a = I32::<4>([1, 2, 3, 4]);
+        let b = I32::<4>([2, 4, 6, 8]);
+        let c = I32::<4>([-1, -2, -3, -4]);
+        let d = I32::<4>([-2, -4, -6, -8]);
 
-        let max = i32x4(MAX, MAX, MAX, MAX);
-        let max1 = i32x4(MAX - 1, MAX - 1, MAX - 1, MAX - 1);
-        let min = i32x4(MIN, MIN, MIN, MIN);
-        let min1 = i32x4(MIN + 1, MIN + 1, MIN + 1, MIN + 1);
+        let max = I32::<4>([MAX, MAX, MAX, MAX]);
+        let max1 = I32::<4>([MAX - 1, MAX - 1, MAX - 1, MAX - 1]);
+        let min = I32::<4>([MIN, MIN, MIN, MIN]);
+        let min1 = I32::<4>([MIN + 1, MIN + 1, MIN + 1, MIN + 1]);
 
-        let z = i32x4(0, 0, 0, 0);
+        let z = I32::<4>([0, 0, 0, 0]);
 
         unsafe {
-            assert_eq!(simd_saturating_add(z, z), z);
-            assert_eq!(simd_saturating_add(z, a), a);
-            assert_eq!(simd_saturating_add(b, z), b);
-            assert_eq!(simd_saturating_add(a, a), b);
-            assert_eq!(simd_saturating_add(a, max), max);
-            assert_eq!(simd_saturating_add(max, b), max);
-            assert_eq!(simd_saturating_add(max1, a), max);
-            assert_eq!(simd_saturating_add(min1, z), min1);
-            assert_eq!(simd_saturating_add(min, z), min);
-            assert_eq!(simd_saturating_add(min1, c), min);
-            assert_eq!(simd_saturating_add(min, c), min);
-            assert_eq!(simd_saturating_add(min1, d), min);
-            assert_eq!(simd_saturating_add(min, d), min);
+            assert_eq!(simd_saturating_add(z, z).0, z.0);
+            assert_eq!(simd_saturating_add(z, a).0, a.0);
+            assert_eq!(simd_saturating_add(b, z).0, b.0);
+            assert_eq!(simd_saturating_add(a, a).0, b.0);
+            assert_eq!(simd_saturating_add(a, max).0, max.0);
+            assert_eq!(simd_saturating_add(max, b).0, max.0);
+            assert_eq!(simd_saturating_add(max1, a).0, max.0);
+            assert_eq!(simd_saturating_add(min1, z).0, min1.0);
+            assert_eq!(simd_saturating_add(min, z).0, min.0);
+            assert_eq!(simd_saturating_add(min1, c).0, min.0);
+            assert_eq!(simd_saturating_add(min, c).0, min.0);
+            assert_eq!(simd_saturating_add(min1, d).0, min.0);
+            assert_eq!(simd_saturating_add(min, d).0, min.0);
 
-            assert_eq!(simd_saturating_sub(b, z), b);
-            assert_eq!(simd_saturating_sub(b, a), a);
-            assert_eq!(simd_saturating_sub(a, a), z);
-            assert_eq!(simd_saturating_sub(a, b), c);
-            assert_eq!(simd_saturating_sub(z, max), min1);
-            assert_eq!(simd_saturating_sub(min1, z), min1);
-            assert_eq!(simd_saturating_sub(min1, a), min);
-            assert_eq!(simd_saturating_sub(min1, b), min);
+            assert_eq!(simd_saturating_sub(b, z).0, b.0);
+            assert_eq!(simd_saturating_sub(b, a).0, a.0);
+            assert_eq!(simd_saturating_sub(a, a).0, z.0);
+            assert_eq!(simd_saturating_sub(a, b).0, c.0);
+            assert_eq!(simd_saturating_sub(z, max).0, min1.0);
+            assert_eq!(simd_saturating_sub(min1, z).0, min1.0);
+            assert_eq!(simd_saturating_sub(min1, a).0, min.0);
+            assert_eq!(simd_saturating_sub(min1, b).0, min.0);
         }
     }
 }
diff --git a/src/test/ui/simd/simd-intrinsic-generic-arithmetic.rs b/src/test/ui/simd/simd-intrinsic-generic-arithmetic.rs
index b67c0ad1eb2..5b0ff88432e 100644
--- a/src/test/ui/simd/simd-intrinsic-generic-arithmetic.rs
+++ b/src/test/ui/simd/simd-intrinsic-generic-arithmetic.rs
@@ -3,7 +3,7 @@
 
 // ignore-emscripten FIXME(#45351) hits an LLVM assert
 
-#![feature(repr_simd, platform_intrinsics)]
+#![feature(repr_simd, platform_intrinsics, min_const_generics)]
 
 #[repr(simd)]
 #[derive(Copy, Clone)]
@@ -11,7 +11,7 @@ struct i32x4(pub i32, pub i32, pub i32, pub i32);
 
 #[repr(simd)]
 #[derive(Copy, Clone)]
-struct u32x4(pub u32, pub u32, pub u32, pub u32);
+struct U32<const N: usize>([u32; N]);
 
 #[repr(simd)]
 #[derive(Copy, Clone)]
@@ -25,6 +25,15 @@ macro_rules! all_eq {
     }}
 }
 
+macro_rules! all_eq_ {
+    ($a: expr, $b: expr) => {{
+        let a = $a;
+        let b = $b;
+        assert!(a.0 == b.0);
+    }}
+}
+
+
 extern "platform-intrinsic" {
     fn simd_add<T>(x: T, y: T) -> T;
     fn simd_sub<T>(x: T, y: T) -> T;
@@ -40,81 +49,81 @@ extern "platform-intrinsic" {
 
 fn main() {
     let x1 = i32x4(1, 2, 3, 4);
-    let y1 = u32x4(1, 2, 3, 4);
+    let y1 = U32::<4>([1, 2, 3, 4]);
     let z1 = f32x4(1.0, 2.0, 3.0, 4.0);
     let x2 = i32x4(2, 3, 4, 5);
-    let y2 = u32x4(2, 3, 4, 5);
+    let y2 = U32::<4>([2, 3, 4, 5]);
     let z2 = f32x4(2.0, 3.0, 4.0, 5.0);
 
     unsafe {
         all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9));
         all_eq!(simd_add(x2, x1), i32x4(3, 5, 7, 9));
-        all_eq!(simd_add(y1, y2), u32x4(3, 5, 7, 9));
-        all_eq!(simd_add(y2, y1), u32x4(3, 5, 7, 9));
+        all_eq_!(simd_add(y1, y2), U32::<4>([3, 5, 7, 9]));
+        all_eq_!(simd_add(y2, y1), U32::<4>([3, 5, 7, 9]));
         all_eq!(simd_add(z1, z2), f32x4(3.0, 5.0, 7.0, 9.0));
         all_eq!(simd_add(z2, z1), f32x4(3.0, 5.0, 7.0, 9.0));
 
         all_eq!(simd_mul(x1, x2), i32x4(2, 6, 12, 20));
         all_eq!(simd_mul(x2, x1), i32x4(2, 6, 12, 20));
-        all_eq!(simd_mul(y1, y2), u32x4(2, 6, 12, 20));
-        all_eq!(simd_mul(y2, y1), u32x4(2, 6, 12, 20));
+        all_eq_!(simd_mul(y1, y2), U32::<4>([2, 6, 12, 20]));
+        all_eq_!(simd_mul(y2, y1), U32::<4>([2, 6, 12, 20]));
         all_eq!(simd_mul(z1, z2), f32x4(2.0, 6.0, 12.0, 20.0));
         all_eq!(simd_mul(z2, z1), f32x4(2.0, 6.0, 12.0, 20.0));
 
         all_eq!(simd_sub(x2, x1), i32x4(1, 1, 1, 1));
         all_eq!(simd_sub(x1, x2), i32x4(-1, -1, -1, -1));
-        all_eq!(simd_sub(y2, y1), u32x4(1, 1, 1, 1));
-        all_eq!(simd_sub(y1, y2), u32x4(!0, !0, !0, !0));
+        all_eq_!(simd_sub(y2, y1), U32::<4>([1, 1, 1, 1]));
+        all_eq_!(simd_sub(y1, y2), U32::<4>([!0, !0, !0, !0]));
         all_eq!(simd_sub(z2, z1), f32x4(1.0, 1.0, 1.0, 1.0));
         all_eq!(simd_sub(z1, z2), f32x4(-1.0, -1.0, -1.0, -1.0));
 
         all_eq!(simd_div(x1, x1), i32x4(1, 1, 1, 1));
         all_eq!(simd_div(i32x4(2, 4, 6, 8), i32x4(2, 2, 2, 2)), x1);
-        all_eq!(simd_div(y1, y1), u32x4(1, 1, 1, 1));
-        all_eq!(simd_div(u32x4(2, 4, 6, 8), u32x4(2, 2, 2, 2)), y1);
+        all_eq_!(simd_div(y1, y1), U32::<4>([1, 1, 1, 1]));
+        all_eq_!(simd_div(U32::<4>([2, 4, 6, 8]), U32::<4>([2, 2, 2, 2])), y1);
         all_eq!(simd_div(z1, z1), f32x4(1.0, 1.0, 1.0, 1.0));
         all_eq!(simd_div(z1, z2), f32x4(1.0/2.0, 2.0/3.0, 3.0/4.0, 4.0/5.0));
         all_eq!(simd_div(z2, z1), f32x4(2.0/1.0, 3.0/2.0, 4.0/3.0, 5.0/4.0));
 
         all_eq!(simd_rem(x1, x1), i32x4(0, 0, 0, 0));
         all_eq!(simd_rem(x2, x1), i32x4(0, 1, 1, 1));
-        all_eq!(simd_rem(y1, y1), u32x4(0, 0, 0, 0));
-        all_eq!(simd_rem(y2, y1), u32x4(0, 1, 1, 1));
+        all_eq_!(simd_rem(y1, y1), U32::<4>([0, 0, 0, 0]));
+        all_eq_!(simd_rem(y2, y1), U32::<4>([0, 1, 1, 1]));
         all_eq!(simd_rem(z1, z1), f32x4(0.0, 0.0, 0.0, 0.0));
         all_eq!(simd_rem(z1, z2), z1);
         all_eq!(simd_rem(z2, z1), f32x4(0.0, 1.0, 1.0, 1.0));
 
         all_eq!(simd_shl(x1, x2), i32x4(1 << 2, 2 << 3, 3 << 4, 4 << 5));
         all_eq!(simd_shl(x2, x1), i32x4(2 << 1, 3 << 2, 4 << 3, 5 << 4));
-        all_eq!(simd_shl(y1, y2), u32x4(1 << 2, 2 << 3, 3 << 4, 4 << 5));
-        all_eq!(simd_shl(y2, y1), u32x4(2 << 1, 3 << 2, 4 << 3, 5 << 4));
+        all_eq_!(simd_shl(y1, y2), U32::<4>([1 << 2, 2 << 3, 3 << 4, 4 << 5]));
+        all_eq_!(simd_shl(y2, y1), U32::<4>([2 << 1, 3 << 2, 4 << 3, 5 << 4]));
 
         // test right-shift by assuming left-shift is correct
         all_eq!(simd_shr(simd_shl(x1, x2), x2), x1);
         all_eq!(simd_shr(simd_shl(x2, x1), x1), x2);
-        all_eq!(simd_shr(simd_shl(y1, y2), y2), y1);
-        all_eq!(simd_shr(simd_shl(y2, y1), y1), y2);
+        all_eq_!(simd_shr(simd_shl(y1, y2), y2), y1);
+        all_eq_!(simd_shr(simd_shl(y2, y1), y1), y2);
 
         // ensure we get logical vs. arithmetic shifts correct
         let (a, b, c, d) = (-12, -123, -1234, -12345);
         all_eq!(simd_shr(i32x4(a, b, c, d), x1), i32x4(a >> 1, b >> 2, c >> 3, d >> 4));
-        all_eq!(simd_shr(u32x4(a as u32, b as u32, c as u32, d as u32), y1),
-                u32x4((a as u32) >> 1, (b as u32) >> 2, (c as u32) >> 3, (d as u32) >> 4));
+        all_eq_!(simd_shr(U32::<4>([a as u32, b as u32, c as u32, d as u32]), y1),
+                U32::<4>([(a as u32) >> 1, (b as u32) >> 2, (c as u32) >> 3, (d as u32) >> 4]));
 
         all_eq!(simd_and(x1, x2), i32x4(0, 2, 0, 4));
         all_eq!(simd_and(x2, x1), i32x4(0, 2, 0, 4));
-        all_eq!(simd_and(y1, y2), u32x4(0, 2, 0, 4));
-        all_eq!(simd_and(y2, y1), u32x4(0, 2, 0, 4));
+        all_eq_!(simd_and(y1, y2), U32::<4>([0, 2, 0, 4]));
+        all_eq_!(simd_and(y2, y1), U32::<4>([0, 2, 0, 4]));
 
         all_eq!(simd_or(x1, x2), i32x4(3, 3, 7, 5));
         all_eq!(simd_or(x2, x1), i32x4(3, 3, 7, 5));
-        all_eq!(simd_or(y1, y2), u32x4(3, 3, 7, 5));
-        all_eq!(simd_or(y2, y1), u32x4(3, 3, 7, 5));
+        all_eq_!(simd_or(y1, y2), U32::<4>([3, 3, 7, 5]));
+        all_eq_!(simd_or(y2, y1), U32::<4>([3, 3, 7, 5]));
 
         all_eq!(simd_xor(x1, x2), i32x4(3, 1, 7, 1));
         all_eq!(simd_xor(x2, x1), i32x4(3, 1, 7, 1));
-        all_eq!(simd_xor(y1, y2), u32x4(3, 1, 7, 1));
-        all_eq!(simd_xor(y2, y1), u32x4(3, 1, 7, 1));
+        all_eq_!(simd_xor(y1, y2), U32::<4>([3, 1, 7, 1]));
+        all_eq_!(simd_xor(y2, y1), U32::<4>([3, 1, 7, 1]));
 
     }
 }