diff options
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 48 | ||||
| -rw-r--r-- | compiler/rustc_hir_analysis/src/check/intrinsic.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_span/src/symbol.rs | 1 | ||||
| -rw-r--r-- | library/core/src/intrinsics/simd.rs | 7 | ||||
| -rw-r--r-- | src/tools/miri/src/intrinsics/simd.rs | 2 | ||||
| -rw-r--r-- | src/tools/miri/tests/pass/intrinsics/portable-simd.rs | 15 | ||||
| -rw-r--r-- | tests/ui/simd/intrinsic/generic-arithmetic-2.rs | 23 | ||||
| -rw-r--r-- | tests/ui/simd/intrinsic/generic-arithmetic-2.stderr | 26 | ||||
| -rw-r--r-- | tests/ui/simd/intrinsic/generic-arithmetic-pass.rs | 10 |
10 files changed, 95 insertions, 40 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 452b5988dab..b17f191ce26 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -348,6 +348,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( | sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz + | sym::simd_ctpop | sym::simd_cttz => { intrinsic_args!(fx, args => (a); intrinsic); @@ -367,6 +368,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( (ty::Uint(_) | ty::Int(_), sym::simd_bswap) => fx.bcx.ins().bswap(lane), (ty::Uint(_) | ty::Int(_), sym::simd_bitreverse) => fx.bcx.ins().bitrev(lane), (ty::Uint(_) | ty::Int(_), sym::simd_ctlz) => fx.bcx.ins().clz(lane), + (ty::Uint(_) | ty::Int(_), sym::simd_ctpop) => fx.bcx.ins().popcnt(lane), (ty::Uint(_) | ty::Int(_), sym::simd_cttz) => fx.bcx.ins().ctz(lane), _ => unreachable!(), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c0a1208a8c7..80e863af893 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -2336,7 +2336,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } // Unary integer intrinsics - if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { + if matches!( + name, + sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz + ) { let vec_ty = bx.cx.type_vector( match *in_elem.kind() { ty::Int(i) => bx.cx.type_int_from_ty(i), @@ -2354,31 +2357,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>( sym::simd_bswap => "bswap", sym::simd_bitreverse => "bitreverse", sym::simd_ctlz => "ctlz", + sym::simd_ctpop => "ctpop", sym::simd_cttz => "cttz", _ => unreachable!(), }; let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); - return if name == sym::simd_bswap && int_size == 8 { + return match name { // byte swap is no-op for i8/u8 - Ok(args[0].immediate()) - } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) { - let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call( - fn_ty, - None, - None, - f, - &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], - None, - None, - )) - } else { - let fn_ty = bx.type_func(&[vec_ty], vec_ty); - let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); - Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()), + sym::simd_ctlz | sym::simd_cttz => { + // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison` + let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); + let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call( + fn_ty, + None, + None, + f, + &[args[0].immediate(), dont_poison_on_zero], + None, + None, + )) + } + sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => { + // simple unary argument cases + let fn_ty = bx.type_func(&[vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None)) + } + _ => unreachable!(), }; } diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index d829e720d9e..50fe20346cf 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -607,6 +607,7 @@ pub fn check_intrinsic_type( | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz + | sym::simd_ctpop | sym::simd_fsqrt | sym::simd_fsin | sym::simd_fcos diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index e453c03e7ce..90671c07938 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1682,6 +1682,7 @@ symbols! { simd_cast_ptr, simd_ceil, simd_ctlz, + simd_ctpop, simd_cttz, simd_div, simd_eq, diff --git a/library/core/src/intrinsics/simd.rs b/library/core/src/intrinsics/simd.rs index ceea6790129..d1be534eaf0 100644 --- a/library/core/src/intrinsics/simd.rs +++ b/library/core/src/intrinsics/simd.rs @@ -569,6 +569,13 @@ extern "rust-intrinsic" { #[rustc_nounwind] pub fn simd_ctlz<T>(x: T) -> T; + /// Count the number of ones in each element. + /// + /// `T` must be a vector of integers. + #[rustc_nounwind] + #[cfg(not(bootstrap))] + pub fn simd_ctpop<T>(x: T) -> T; + /// Count the trailing zeros of each element. /// /// `T` must be a vector of integers. diff --git a/src/tools/miri/src/intrinsics/simd.rs b/src/tools/miri/src/intrinsics/simd.rs index 4cde364fbc4..e178187ffa6 100644 --- a/src/tools/miri/src/intrinsics/simd.rs +++ b/src/tools/miri/src/intrinsics/simd.rs @@ -42,6 +42,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { | "flog2" | "flog10" | "ctlz" + | "ctpop" | "cttz" | "bswap" | "bitreverse" @@ -68,6 +69,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway), "trunc" => Op::Round(rustc_apfloat::Round::TowardZero), "ctlz" => Op::Numeric(sym::ctlz), + "ctpop" => Op::Numeric(sym::ctpop), "cttz" => Op::Numeric(sym::cttz), "bswap" => Op::Numeric(sym::bswap), "bitreverse" => Op::Numeric(sym::bitreverse), diff --git a/src/tools/miri/tests/pass/intrinsics/portable-simd.rs b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs index 1fc713d48dc..248a57d6850 100644 --- a/src/tools/miri/tests/pass/intrinsics/portable-simd.rs +++ b/src/tools/miri/tests/pass/intrinsics/portable-simd.rs @@ -506,6 +506,21 @@ fn simd_intrinsics() { assert!(!simd_reduce_all(i32x2::from_array([0, -1]))); assert_eq!( + simd_ctlz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])), + i32x4::from_array([32, 1, 0, 0]) + ); + + assert_eq!( + simd_ctpop(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])), + i32x4::from_array([0, 31, 1, 32]) + ); + + assert_eq!( + simd_cttz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])), + i32x4::from_array([32, 0, 31, 0]) + ); + + assert_eq!( simd_select(i8x4::from_array([0, -1, -1, 0]), a, b), i32x4::from_array([1, 10, 10, 4]) ); diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs index 4ad98d56711..fc3087cbf75 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-2.rs +++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.rs @@ -30,6 +30,7 @@ extern "rust-intrinsic" { fn simd_bswap<T>(x: T) -> T; fn simd_bitreverse<T>(x: T) -> T; fn simd_ctlz<T>(x: T) -> T; + fn simd_ctpop<T>(x: T) -> T; fn simd_cttz<T>(x: T) -> T; } @@ -77,7 +78,6 @@ fn main() { simd_cttz(x); simd_cttz(y); - simd_add(0, 0); //~^ ERROR expected SIMD input type, found non-SIMD `i32` simd_sub(0, 0); @@ -108,24 +108,25 @@ fn main() { simd_cttz(0); //~^ ERROR expected SIMD input type, found non-SIMD `i32` - simd_shl(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_shr(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_and(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_or(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_xor(z, z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_bswap(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_bitreverse(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_ctlz(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` + simd_ctpop(z); + //~^ ERROR unsupported operation on `f32x4` with element `f32` simd_cttz(z); -//~^ ERROR unsupported operation on `f32x4` with element `f32` + //~^ ERROR unsupported operation on `f32x4` with element `f32` } } diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr index db26f3417c9..6f5f86d7d37 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr +++ b/tests/ui/simd/intrinsic/generic-arithmetic-2.stderr @@ -83,59 +83,65 @@ LL | simd_cttz(0); | ^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:112:9 + --> $DIR/generic-arithmetic-2.rs:111:9 | LL | simd_shl(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:114:9 + --> $DIR/generic-arithmetic-2.rs:113:9 | LL | simd_shr(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:116:9 + --> $DIR/generic-arithmetic-2.rs:115:9 | LL | simd_and(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:118:9 + --> $DIR/generic-arithmetic-2.rs:117:9 | LL | simd_or(z, z); | ^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:120:9 + --> $DIR/generic-arithmetic-2.rs:119:9 | LL | simd_xor(z, z); | ^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:122:9 + --> $DIR/generic-arithmetic-2.rs:121:9 | LL | simd_bswap(z); | ^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:124:9 + --> $DIR/generic-arithmetic-2.rs:123:9 | LL | simd_bitreverse(z); | ^^^^^^^^^^^^^^^^^^ error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:126:9 + --> $DIR/generic-arithmetic-2.rs:125:9 | LL | simd_ctlz(z); | ^^^^^^^^^^^^ +error[E0511]: invalid monomorphization of `simd_ctpop` intrinsic: unsupported operation on `f32x4` with element `f32` + --> $DIR/generic-arithmetic-2.rs:127:9 + | +LL | simd_ctpop(z); + | ^^^^^^^^^^^^^ + error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32` - --> $DIR/generic-arithmetic-2.rs:128:9 + --> $DIR/generic-arithmetic-2.rs:129:9 | LL | simd_cttz(z); | ^^^^^^^^^^^^ -error: aborting due to 23 previous errors +error: aborting due to 24 previous errors For more information about this error, try `rustc --explain E0511`. diff --git a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs index 33143b1f7b5..60dfa627414 100644 --- a/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs +++ b/tests/ui/simd/intrinsic/generic-arithmetic-pass.rs @@ -47,6 +47,7 @@ extern "rust-intrinsic" { fn simd_bswap<T>(x: T) -> T; fn simd_bitreverse<T>(x: T) -> T; fn simd_ctlz<T>(x: T) -> T; + fn simd_ctpop<T>(x: T) -> T; fn simd_cttz<T>(x: T) -> T; } @@ -57,6 +58,8 @@ fn main() { let x2 = i32x4(2, 3, 4, 5); let y2 = U32::<4>([2, 3, 4, 5]); let z2 = f32x4(2.0, 3.0, 4.0, 5.0); + let x3 = i32x4(0, i32::MAX, i32::MIN, -1_i32); + let y3 = U32::<4>([0, i32::MAX as _, i32::MIN as _, -1_i32 as _]); unsafe { all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9)); @@ -147,6 +150,13 @@ fn main() { all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29)); all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29])); + all_eq!(simd_ctpop(x1), i32x4(1, 1, 2, 1)); + all_eq_!(simd_ctpop(y1), U32::<4>([1, 1, 2, 1])); + all_eq!(simd_ctpop(x2), i32x4(1, 2, 1, 2)); + all_eq_!(simd_ctpop(y2), U32::<4>([1, 2, 1, 2])); + all_eq!(simd_ctpop(x3), i32x4(0, 31, 1, 32)); + all_eq_!(simd_ctpop(y3), U32::<4>([0, 31, 1, 32])); + all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2)); all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2])); } |