Add SIMD bitreverse, ctlz, cttz intrinsics

6 files changed, 117 insertions, 25 deletions

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index c30cb6801da..187214a93dd 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -2074,7 +2074,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         simd_neg: Int => neg, Float => fneg;
     }
 
-    if name == sym::simd_bswap {
+    // Unary integer intrinsics
+    if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) {
         let vec_ty = bx.cx.type_vector(
             match *in_elem.kind() {
                 ty::Int(i) => bx.cx.type_int_from_ty(i),
@@ -2088,12 +2089,29 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             },
             in_len as u64,
         );
-        let llvm_intrinsic =
-            &format!("llvm.bswap.v{}i{}", in_len, in_elem.int_size_and_signed(bx.tcx()).0.bits(),);
-        let fn_ty = bx.type_func(&[vec_ty], vec_ty);
-        let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
-        let v = bx.call(fn_ty, None, None, f, &[args[0].immediate()], None);
-        return Ok(v);
+        let intrinsic_name = match name {
+            sym::simd_bswap => "bswap",
+            sym::simd_bitreverse => "bitreverse",
+            sym::simd_ctlz => "ctlz",
+            sym::simd_cttz => "cttz",
+            _ => unreachable!(),
+        };
+        let llvm_intrinsic = &format!(
+            "llvm.{}.v{}i{}",
+            intrinsic_name,
+            in_len,
+            in_elem.int_size_and_signed(bx.tcx()).0.bits(),
+        );
+
+        return Ok(if matches!(name, sym::simd_ctlz | sym::simd_cttz) {
+            let fn_ty = bx.type_func(&[vec_ty, bx.type_bool()], vec_ty);
+            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+            bx.call(fn_ty, None, None, f, &[args[0].immediate(), bx.const_bool(false)], None)
+        } else {
+            let fn_ty = bx.type_func(&[vec_ty], vec_ty);
+            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+            bx.call(fn_ty, None, None, f, &[args[0].immediate()], None)
+        });
     }
 
     if name == sym::simd_arith_offset {
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index 68c99c4f24e..064021b1ea4 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -522,6 +522,9 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         sym::simd_arith_offset => (2, vec![param(0), param(1)], param(0)),
         sym::simd_neg
         | sym::simd_bswap
+        | sym::simd_bitreverse
+        | sym::simd_ctlz
+        | sym::simd_cttz
         | sym::simd_fsqrt
         | sym::simd_fsin
         | sym::simd_fcos
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 445bcd58426..35f8b99ffb2 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1369,10 +1369,13 @@ symbols! {
         simd_arith_offset,
         simd_as,
         simd_bitmask,
+        simd_bitreverse,
         simd_bswap,
         simd_cast,
         simd_cast_ptr,
         simd_ceil,
+        simd_ctlz,
+        simd_cttz,
         simd_div,
         simd_eq,
         simd_expose_addr,
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
index 310d5cab843..62fb5238bbd 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
@@ -28,6 +28,9 @@ extern "platform-intrinsic" {
 
     fn simd_neg<T>(x: T) -> T;
     fn simd_bswap<T>(x: T) -> T;
+    fn simd_bitreverse<T>(x: T) -> T;
+    fn simd_ctlz<T>(x: T) -> T;
+    fn simd_cttz<T>(x: T) -> T;
 }
 
 fn main() {
@@ -67,6 +70,12 @@ fn main() {
         simd_neg(z);
         simd_bswap(x);
         simd_bswap(y);
+        simd_bitreverse(x);
+        simd_bitreverse(y);
+        simd_ctlz(x);
+        simd_ctlz(y);
+        simd_cttz(x);
+        simd_cttz(y);
 
 
         simd_add(0, 0);
@@ -92,6 +101,12 @@ fn main() {
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_bswap(0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
+        simd_bitreverse(0);
+        //~^ ERROR expected SIMD input type, found non-SIMD `i32`
+        simd_ctlz(0);
+        //~^ ERROR expected SIMD input type, found non-SIMD `i32`
+        simd_cttz(0);
+        //~^ ERROR expected SIMD input type, found non-SIMD `i32`
 
 
         simd_shl(z, z);
@@ -106,5 +121,11 @@ fn main() {
 //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_bswap(z);
 //~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_bitreverse(z);
+//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_ctlz(z);
+//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_cttz(z);
+//~^ ERROR unsupported operation on `f32x4` with element `f32`
     }
 }
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
index efc6f05d39d..db26f3417c9 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
@@ -1,105 +1,141 @@
 error[E0511]: invalid monomorphization of `simd_add` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:72:9
+  --> $DIR/generic-arithmetic-2.rs:81:9
    |
 LL |         simd_add(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_sub` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:74:9
+  --> $DIR/generic-arithmetic-2.rs:83:9
    |
 LL |         simd_sub(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_mul` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:76:9
+  --> $DIR/generic-arithmetic-2.rs:85:9
    |
 LL |         simd_mul(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_div` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:78:9
+  --> $DIR/generic-arithmetic-2.rs:87:9
    |
 LL |         simd_div(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shl` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:80:9
+  --> $DIR/generic-arithmetic-2.rs:89:9
    |
 LL |         simd_shl(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shr` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:82:9
+  --> $DIR/generic-arithmetic-2.rs:91:9
    |
 LL |         simd_shr(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_and` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:84:9
+  --> $DIR/generic-arithmetic-2.rs:93:9
    |
 LL |         simd_and(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_or` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:86:9
+  --> $DIR/generic-arithmetic-2.rs:95:9
    |
 LL |         simd_or(0, 0);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_xor` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:88:9
+  --> $DIR/generic-arithmetic-2.rs:97:9
    |
 LL |         simd_xor(0, 0);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_neg` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:91:9
+  --> $DIR/generic-arithmetic-2.rs:100:9
    |
 LL |         simd_neg(0);
    |         ^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: expected SIMD input type, found non-SIMD `i32`
-  --> $DIR/generic-arithmetic-2.rs:93:9
+  --> $DIR/generic-arithmetic-2.rs:102:9
    |
 LL |         simd_bswap(0);
    |         ^^^^^^^^^^^^^
 
+error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: expected SIMD input type, found non-SIMD `i32`
+  --> $DIR/generic-arithmetic-2.rs:104:9
+   |
+LL |         simd_bitreverse(0);
+   |         ^^^^^^^^^^^^^^^^^^
+
+error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: expected SIMD input type, found non-SIMD `i32`
+  --> $DIR/generic-arithmetic-2.rs:106:9
+   |
+LL |         simd_ctlz(0);
+   |         ^^^^^^^^^^^^
+
+error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: expected SIMD input type, found non-SIMD `i32`
+  --> $DIR/generic-arithmetic-2.rs:108:9
+   |
+LL |         simd_cttz(0);
+   |         ^^^^^^^^^^^^
+
 error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:97:9
+  --> $DIR/generic-arithmetic-2.rs:112:9
    |
 LL |         simd_shl(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:99:9
+  --> $DIR/generic-arithmetic-2.rs:114:9
    |
 LL |         simd_shr(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:101:9
+  --> $DIR/generic-arithmetic-2.rs:116:9
    |
 LL |         simd_and(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:103:9
+  --> $DIR/generic-arithmetic-2.rs:118:9
    |
 LL |         simd_or(z, z);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:105:9
+  --> $DIR/generic-arithmetic-2.rs:120:9
    |
 LL |         simd_xor(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:107:9
+  --> $DIR/generic-arithmetic-2.rs:122:9
    |
 LL |         simd_bswap(z);
    |         ^^^^^^^^^^^^^
 
-error: aborting due to 17 previous errors
+error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32`
+  --> $DIR/generic-arithmetic-2.rs:124:9
+   |
+LL |         simd_bitreverse(z);
+   |         ^^^^^^^^^^^^^^^^^^
+
+error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32`
+  --> $DIR/generic-arithmetic-2.rs:126:9
+   |
+LL |         simd_ctlz(z);
+   |         ^^^^^^^^^^^^
+
+error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32`
+  --> $DIR/generic-arithmetic-2.rs:128:9
+   |
+LL |         simd_cttz(z);
+   |         ^^^^^^^^^^^^
+
+error: aborting due to 23 previous errors
 
 For more information about this error, try `rustc --explain E0511`.
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
index ca3bae99fea..1168e6aae0f 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
@@ -48,6 +48,9 @@ extern "platform-intrinsic" {
 
     fn simd_neg<T>(x: T) -> T;
     fn simd_bswap<T>(x: T) -> T;
+    fn simd_bitreverse<T>(x: T) -> T;
+    fn simd_ctlz<T>(x: T) -> T;
+    fn simd_cttz<T>(x: T) -> T;
 }
 
 fn main() {
@@ -136,5 +139,13 @@ fn main() {
         all_eq!(simd_bswap(x1), i32x4(0x01000000, 0x02000000, 0x03000000, 0x04000000));
         all_eq_!(simd_bswap(y1), U32::<4>([0x01000000, 0x02000000, 0x03000000, 0x04000000]));
 
+        all_eq!(simd_bitreverse(x1), i32x4(0x80000000u32 as i32, 0x40000000, 0xc0000000u32 as i32, 0x20000000));
+        all_eq_!(simd_bitreverse(y1), U32::<4>([0x80000000, 0x40000000, 0xc0000000, 0x20000000]));
+
+        all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29));
+        all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29]));
+
+        all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2));
+        all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2]));
     }
 }