compiler: add simd_ctpop intrinsic

7 files changed, 76 insertions, 40 deletions

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index c0a1208a8c7..7e6d7491798 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -2336,7 +2336,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     // Unary integer intrinsics
-    if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) {
+    if matches!(
+        name,
+        sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
+    ) {
         let vec_ty = bx.cx.type_vector(
             match *in_elem.kind() {
                 ty::Int(i) => bx.cx.type_int_from_ty(i),
@@ -2354,31 +2357,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             sym::simd_bswap => "bswap",
             sym::simd_bitreverse => "bitreverse",
             sym::simd_ctlz => "ctlz",
+            sym::simd_ctpop => "ctpop",
             sym::simd_cttz => "cttz",
             _ => unreachable!(),
         };
         let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
         let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
 
-        return if name == sym::simd_bswap && int_size == 8 {
+        return match name {
             // byte swap is no-op for i8/u8
-            Ok(args[0].immediate())
-        } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) {
-            let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
-            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
-            Ok(bx.call(
-                fn_ty,
-                None,
-                None,
-                f,
-                &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)],
-                None,
-                None,
-            ))
-        } else {
-            let fn_ty = bx.type_func(&[vec_ty], vec_ty);
-            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
-            Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
+            sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
+            sym::simd_ctlz | sym::simd_cttz => {
+                // this fun bonus i1 arg means "poison if the arg vector contains zero"
+                let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
+                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+                Ok(bx.call(
+                    fn_ty,
+                    None,
+                    None,
+                    f,
+                    // simd_ctlz and simd_cttz are exposed to safe code, so let's not poison anything
+                    &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)],
+                    None,
+                    None,
+                ))
+            }
+            sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
+                // simple unary argument cases
+                let fn_ty = bx.type_func(&[vec_ty], vec_ty);
+                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+                Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
+            }
+            _ => unreachable!(),
         };
     }
 
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index d829e720d9e..50fe20346cf 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -607,6 +607,7 @@ pub fn check_intrinsic_type(
             | sym::simd_bitreverse
             | sym::simd_ctlz
             | sym::simd_cttz
+            | sym::simd_ctpop
             | sym::simd_fsqrt
             | sym::simd_fsin
             | sym::simd_fcos
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 68b1b32baf2..203d24185c6 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1681,6 +1681,7 @@ symbols! {
         simd_cast_ptr,
         simd_ceil,
         simd_ctlz,
+        simd_ctpop,
         simd_cttz,
         simd_div,
         simd_eq,
diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs
index ceea6790129..d1be534eaf0 100644
--- a/library/core/src/intrinsics/simd.rs
+++ b/library/core/src/intrinsics/simd.rs
@@ -569,6 +569,13 @@ extern "rust-intrinsic" {
     #[rustc_nounwind]
     pub fn simd_ctlz<T>(x: T) -> T;
 
+    /// Count the number of ones in each element.
+    ///
+    /// `T` must be a vector of integers.
+    #[rustc_nounwind]
+    #[cfg(not(bootstrap))]
+    pub fn simd_ctpop<T>(x: T) -> T;
+
     /// Count the trailing zeros of each element.
     ///
     /// `T` must be a vector of integers.
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
index 4ad98d56711..fc3087cbf75 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs
@@ -30,6 +30,7 @@ extern "rust-intrinsic" {
     fn simd_bswap<T>(x: T) -> T;
     fn simd_bitreverse<T>(x: T) -> T;
     fn simd_ctlz<T>(x: T) -> T;
+    fn simd_ctpop<T>(x: T) -> T;
     fn simd_cttz<T>(x: T) -> T;
 }
 
@@ -77,7 +78,6 @@ fn main() {
         simd_cttz(x);
         simd_cttz(y);
 
-
         simd_add(0, 0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_sub(0, 0);
@@ -108,24 +108,25 @@ fn main() {
         simd_cttz(0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
 
-
         simd_shl(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_shr(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_and(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_or(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_xor(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_bswap(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_bitreverse(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_ctlz(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_ctpop(z);
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_cttz(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
     }
 }
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
index db26f3417c9..6f5f86d7d37 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr
@@ -83,59 +83,65 @@ LL |         simd_cttz(0);
    |         ^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:112:9
+  --> $DIR/generic-arithmetic-2.rs:111:9
    |
 LL |         simd_shl(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:114:9
+  --> $DIR/generic-arithmetic-2.rs:113:9
    |
 LL |         simd_shr(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:116:9
+  --> $DIR/generic-arithmetic-2.rs:115:9
    |
 LL |         simd_and(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:118:9
+  --> $DIR/generic-arithmetic-2.rs:117:9
    |
 LL |         simd_or(z, z);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:120:9
+  --> $DIR/generic-arithmetic-2.rs:119:9
    |
 LL |         simd_xor(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:122:9
+  --> $DIR/generic-arithmetic-2.rs:121:9
    |
 LL |         simd_bswap(z);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:124:9
+  --> $DIR/generic-arithmetic-2.rs:123:9
    |
 LL |         simd_bitreverse(z);
    |         ^^^^^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:126:9
+  --> $DIR/generic-arithmetic-2.rs:125:9
    |
 LL |         simd_ctlz(z);
    |         ^^^^^^^^^^^^
 
+error[E0511]: invalid monomorphization of `simd_ctpop` intrinsic: unsupported operation on `f32x4` with element `f32`
+  --> $DIR/generic-arithmetic-2.rs:127:9
+   |
+LL |         simd_ctpop(z);
+   |         ^^^^^^^^^^^^^
+
 error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:128:9
+  --> $DIR/generic-arithmetic-2.rs:129:9
    |
 LL |         simd_cttz(z);
    |         ^^^^^^^^^^^^
 
-error: aborting due to 23 previous errors
+error: aborting due to 24 previous errors
 
 For more information about this error, try `rustc --explain E0511`.
diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
index 33143b1f7b5..60dfa627414 100644
--- a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
+++ b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
@@ -47,6 +47,7 @@ extern "rust-intrinsic" {
     fn simd_bswap<T>(x: T) -> T;
     fn simd_bitreverse<T>(x: T) -> T;
     fn simd_ctlz<T>(x: T) -> T;
+    fn simd_ctpop<T>(x: T) -> T;
     fn simd_cttz<T>(x: T) -> T;
 }
 
@@ -57,6 +58,8 @@ fn main() {
     let x2 = i32x4(2, 3, 4, 5);
     let y2 = U32::<4>([2, 3, 4, 5]);
     let z2 = f32x4(2.0, 3.0, 4.0, 5.0);
+    let x3 = i32x4(0, i32::MAX, i32::MIN, -1_i32);
+    let y3 = U32::<4>([0, i32::MAX as _, i32::MIN as _, -1_i32 as _]);
 
     unsafe {
         all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9));
@@ -147,6 +150,13 @@ fn main() {
         all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29));
         all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29]));
 
+        all_eq!(simd_ctpop(x1), i32x4(1, 1, 2, 1));
+        all_eq_!(simd_ctpop(y1), U32::<4>([1, 1, 2, 1]));
+        all_eq!(simd_ctpop(x2), i32x4(1, 2, 1, 2));
+        all_eq_!(simd_ctpop(y2), U32::<4>([1, 2, 1, 2]));
+        all_eq!(simd_ctpop(x3), i32x4(0, 31, 1, 32));
+        all_eq_!(simd_ctpop(y3), U32::<4>([0, 31, 1, 32]));
+
         all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2));
         all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2]));
     }