From f283e449b11ebe8127570aab09b8871442d1e74b Mon Sep 17 00:00:00 2001 From: James Barford-Evans Date: Wed, 18 Dec 2024 15:42:53 +0000 Subject: PR feedback & pipeline --- .../crates/core_arch/src/aarch64/neon/generated.rs | 2 +- .../core_arch/src/arm_shared/neon/generated.rs | 109 +- .../crates/core_arch/src/arm_shared/neon/mod.rs | 73 +- library/stdarch/crates/stdarch-gen-arm/Cargo.toml | 23 + .../stdarch-gen-arm/spec/neon/aarch64.spec.yml | 9852 ++++++++++++++++++++ .../stdarch-gen-arm/spec/neon/arm_shared.spec.yml | 9789 +++++++++++++++++++ .../crates/stdarch-gen-arm/src/assert_instr.rs | 372 + .../stdarch/crates/stdarch-gen-arm/src/context.rs | 263 + .../crates/stdarch-gen-arm/src/expression.rs | 576 ++ .../crates/stdarch-gen-arm/src/fn_suffix.rs | 308 + .../stdarch/crates/stdarch-gen-arm/src/input.rs | 432 + .../crates/stdarch-gen-arm/src/intrinsic.rs | 1624 ++++ .../crates/stdarch-gen-arm/src/load_store_tests.rs | 818 ++ library/stdarch/crates/stdarch-gen-arm/src/main.rs | 279 + .../stdarch/crates/stdarch-gen-arm/src/matching.rs | 170 + .../crates/stdarch-gen-arm/src/predicate_forms.rs | 249 + .../crates/stdarch-gen-arm/src/typekinds.rs | 1051 +++ .../crates/stdarch-gen-arm/src/wildcards.rs | 197 + .../crates/stdarch-gen-arm/src/wildstring.rs | 399 + library/stdarch/crates/stdarch-gen2/Cargo.toml | 22 - .../crates/stdarch-gen2/spec/neon/aarch64.spec.yml | 9852 -------------------- .../stdarch-gen2/spec/neon/arm_shared.spec.yml | 9789 ------------------- .../crates/stdarch-gen2/src/assert_instr.rs | 372 - library/stdarch/crates/stdarch-gen2/src/context.rs | 263 - .../stdarch/crates/stdarch-gen2/src/expression.rs | 576 -- .../stdarch/crates/stdarch-gen2/src/fn_suffix.rs | 308 - library/stdarch/crates/stdarch-gen2/src/input.rs | 432 - .../stdarch/crates/stdarch-gen2/src/intrinsic.rs | 1624 ---- .../crates/stdarch-gen2/src/load_store_tests.rs | 818 -- library/stdarch/crates/stdarch-gen2/src/main.rs | 279 - .../stdarch/crates/stdarch-gen2/src/matching.rs | 170 - .../crates/stdarch-gen2/src/predicate_forms.rs | 249 - .../stdarch/crates/stdarch-gen2/src/typekinds.rs | 1051 --- .../stdarch/crates/stdarch-gen2/src/wildcards.rs | 197 - .../stdarch/crates/stdarch-gen2/src/wildstring.rs | 399 - library/stdarch/crates/stdarch-verify/src/lib.rs | 88 +- library/stdarch/crates/stdarch-verify/tests/arm.rs | 172 +- 37 files changed, 26507 insertions(+), 26740 deletions(-) create mode 100644 library/stdarch/crates/stdarch-gen-arm/Cargo.toml create mode 100644 library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml create mode 100644 library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/assert_instr.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/context.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/expression.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/fn_suffix.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/input.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/main.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/matching.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/predicate_forms.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/typekinds.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/wildcards.rs create mode 100644 library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/Cargo.toml delete mode 100644 library/stdarch/crates/stdarch-gen2/spec/neon/aarch64.spec.yml delete mode 100644 library/stdarch/crates/stdarch-gen2/spec/neon/arm_shared.spec.yml delete mode 100644 library/stdarch/crates/stdarch-gen2/src/assert_instr.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/context.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/expression.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/fn_suffix.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/input.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/intrinsic.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/load_store_tests.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/main.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/matching.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/predicate_forms.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/typekinds.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/wildcards.rs delete mode 100644 library/stdarch/crates/stdarch-gen2/src/wildstring.rs (limited to 'library/stdarch/crates') diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index 08f7aecdb62..cedc0c458b0 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -7953,7 +7953,7 @@ pub unsafe fn vcvts_n_f32_u32(a: u32) -> f32 { #[rustc_legacy_const_generics(1)] #[stable(feature = "neon_intrinsics", since = "1.59.0")] pub unsafe fn vcvtd_n_f64_u64(a: u64) -> f64 { - static_assert!(N >= 1 && N <= 32); + static_assert!(N >= 1 && N <= 64); extern "unadjusted" { #[cfg_attr( any(target_arch = "aarch64", target_arch = "arm64ec"), diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs index e89067142b3..09be5282e9e 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs @@ -15227,7 +15227,7 @@ pub unsafe fn vld4_lane_f32(a: *const f32, b: float32x2x4_t) -> size: i32, ) -> float32x2x4_t; } - vld4_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vld4_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15254,7 +15254,7 @@ pub unsafe fn vld4q_lane_f32(a: *const f32, b: float32x4x4_t) - size: i32, ) -> float32x4x4_t; } - vld4q_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vld4q_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15281,7 +15281,7 @@ pub unsafe fn vld4_lane_s8(a: *const i8, b: int8x8x4_t) -> int8 size: i32, ) -> int8x8x4_t; } - vld4_lane_s8(a as _, b.0, b.1, b.2, b.3, LANE, 1) + _vld4_lane_s8(a as _, b.0, b.1, b.2, b.3, LANE, 1) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15308,7 +15308,7 @@ pub unsafe fn vld4_lane_s16(a: *const i16, b: int16x4x4_t) -> i size: i32, ) -> int16x4x4_t; } - vld4_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) + _vld4_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15335,7 +15335,7 @@ pub unsafe fn vld4q_lane_s16(a: *const i16, b: int16x8x4_t) -> size: i32, ) -> int16x8x4_t; } - vld4q_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) + _vld4q_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15362,7 +15362,7 @@ pub unsafe fn vld4_lane_s32(a: *const i32, b: int32x2x4_t) -> i size: i32, ) -> int32x2x4_t; } - vld4_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vld4_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Load multiple 4-element structures to four registers"] @@ -15389,7 +15389,7 @@ pub unsafe fn vld4q_lane_s32(a: *const i32, b: int32x4x4_t) -> size: i32, ) -> int32x4x4_t; } - vld4q_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vld4q_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Load multiple 4-element structures to four registers"] @@ -25928,7 +25928,7 @@ pub unsafe fn vqrshrn_n_s32(a: int32x4_t) -> int16x4_t { } _vqrshrn_n_s32( a, - const { int16x8_t([-N as i32, -N as i32, -N as i32, -N as i32]) }, + const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }, ) } @@ -25948,7 +25948,7 @@ pub unsafe fn vqrshrn_n_s64(a: int64x2_t) -> int32x2_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftns.v2i32")] fn _vqrshrn_n_s64(a: int64x2_t, n: int64x2_t) -> int32x2_t; } - _vqrshrn_n_s64(a, const { int16x8_t([-N as i64, -N as i64]) }) + _vqrshrn_n_s64(a, const { int64x2_t([-N as i64, -N as i64]) }) } #[doc = "Signed saturating rounded shift right narrow"] @@ -26033,15 +26033,17 @@ pub unsafe fn vqrshrn_n_u16(a: uint16x8_t) -> uint8x8_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v8i8")] fn _vqrshrn_n_u16(a: int16x8_t, n: int16x8_t) -> int8x8_t; } - _vqrshrnq_n_u16( + _vqrshrn_n_u16( a.as_signed(), const { uint16x8_t([ -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, ]) - }, + } + .as_signed(), ) + .as_unsigned() } #[doc = "Unsigned signed saturating rounded shift right narrow"] @@ -26060,10 +26062,11 @@ pub unsafe fn vqrshrn_n_u32(a: uint32x4_t) -> uint16x4_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v4i16")] fn _vqrshrn_n_u32(a: int32x4_t, n: int32x4_t) -> int16x4_t; } - _vqrshrnq_n_u32( + _vqrshrn_n_u32( a.as_signed(), - const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }, + const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }.as_signed(), ) + .as_unsigned() } #[doc = "Unsigned signed saturating rounded shift right narrow"] @@ -26082,7 +26085,11 @@ pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnu.v2i32")] fn _vqrshrn_n_u64(a: int64x2_t, n: int64x2_t) -> int32x2_t; } - _vqrshrnq_n_u64(a.as_signed(), const { uint64x2_t([-N as u64, -N as u64]) }) + _vqrshrn_n_u64( + a.as_signed(), + const { uint64x2_t([-N as u64, -N as u64]) }.as_signed(), + ) + .as_unsigned() } #[doc = "Unsigned signed saturating rounded shift right narrow"] @@ -26162,14 +26169,7 @@ pub unsafe fn vqrshrn_n_u64(a: uint64x2_t) -> uint32x2_t { #[rustc_legacy_const_generics(1)] #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { - static_assert!( - const { - int16x8_t([ - -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, - -N as i16, - ]) - } - ); + static_assert!(N >= 1 && N <= 8); extern "unadjusted" { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v8i8")] fn _vqrshrun_n_s16(a: int16x8_t, n: int16x8_t) -> int8x8_t; @@ -26197,7 +26197,7 @@ pub unsafe fn vqrshrun_n_s16(a: int16x8_t) -> uint8x8_t { #[rustc_legacy_const_generics(1)] #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { - static_assert!(const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }); + static_assert!(N >= 1 && N <= 16); extern "unadjusted" { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v4i16")] fn _vqrshrun_n_s32(a: int32x4_t, n: int32x4_t) -> int16x4_t; @@ -26220,7 +26220,7 @@ pub unsafe fn vqrshrun_n_s32(a: int32x4_t) -> uint16x4_t { #[rustc_legacy_const_generics(1)] #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] pub unsafe fn vqrshrun_n_s64(a: int64x2_t) -> uint32x2_t { - static_assert!(const { int64x2_t([-N as i64, -N as i64]) }); + static_assert!(N >= 1 && N <= 32); extern "unadjusted" { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqrshiftnsu.v2i32")] fn _vqrshrun_n_s64(a: int64x2_t, n: int64x2_t) -> int32x2_t; @@ -27773,7 +27773,8 @@ pub unsafe fn vqshrn_n_u16(a: uint16x8_t) -> uint8x8_t { -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, ]) - }, + } + .as_signed(), ) .as_unsigned() } @@ -27796,7 +27797,7 @@ pub unsafe fn vqshrn_n_u32(a: uint32x4_t) -> uint16x4_t { } _vqshrn_n_u32( a.as_signed(), - const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }, + const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }.as_signed(), ) .as_unsigned() } @@ -27817,7 +27818,11 @@ pub unsafe fn vqshrn_n_u64(a: uint64x2_t) -> uint32x2_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnu.v2i32")] fn _vqshrn_n_u64(a: int64x2_t, n: int64x2_t) -> int32x2_t; } - _vqshrn_n_u64(a.as_signed(), const { uint64x2_t([-N as u64, -N as u64]) }).as_unsigned() + _vqshrn_n_u64( + a.as_signed(), + const { uint64x2_t([-N as u64, -N as u64]) }.as_signed(), + ) + .as_unsigned() } #[doc = "Unsigned saturating shift right narrow"] @@ -27902,7 +27907,7 @@ pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v8i8")] fn _vqshrun_n_s16(a: int16x8_t, n: int16x8_t) -> int8x8_t; } - _vqshrun_n_u8( + _vqshrun_n_s16( a, const { int16x8_t([ @@ -27911,6 +27916,7 @@ pub unsafe fn vqshrun_n_s16(a: int16x8_t) -> uint8x8_t { ]) }, ) + .as_unsigned() } #[doc = "Signed saturating shift right unsigned narrow"] @@ -27929,10 +27935,11 @@ pub unsafe fn vqshrun_n_s32(a: int32x4_t) -> uint16x4_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v4i16")] fn _vqshrun_n_s32(a: int32x4_t, n: int32x4_t) -> int16x4_t; } - _vqshrun_n_u16( + _vqshrun_n_s32( a, const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }, ) + .as_unsigned() } #[doc = "Signed saturating shift right unsigned narrow"] @@ -27951,7 +27958,7 @@ pub unsafe fn vqshrun_n_s64(a: int64x2_t) -> uint32x2_t { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vqshiftnsu.v2i32")] fn _vqshrun_n_s64(a: int64x2_t, n: int64x2_t) -> int32x2_t; } - _vqshrun_n_u32(a, const { int64x2_t([-N as i64, -N as i64]) }) + _vqshrun_n_s64(a, const { int64x2_t([-N as i64, -N as i64]) }).as_unsigned() } #[doc = "Signed saturating shift right unsigned narrow"] @@ -43688,7 +43695,7 @@ pub unsafe fn vst2_lane_f32(a: *mut f32, b: float32x2x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v2f32")] fn _vst2_lane_f32(ptr: *mut i8, a: float32x2_t, b: float32x2_t, n: i32, size: i32); } - _vst2_f32(a as _, b.0, b.1, LANE, 4) + _vst2_lane_f32(a as _, b.0, b.1, LANE, 4) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43707,7 +43714,7 @@ pub unsafe fn vst2q_lane_f32(a: *mut f32, b: float32x4x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4f32")] fn _vst2q_lane_f32(ptr: *mut i8, a: float32x4_t, b: float32x4_t, n: i32, size: i32); } - _vst2q_f32(a as _, b.0, b.1, LANE, 4) + _vst2q_lane_f32(a as _, b.0, b.1, LANE, 4) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43726,7 +43733,7 @@ pub unsafe fn vst2_lane_s8(a: *mut i8, b: int8x8x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v8i8")] fn _vst2_lane_s8(ptr: *mut i8, a: int8x8_t, b: int8x8_t, n: i32, size: i32); } - _vst2_s8(a as _, b.0, b.1, LANE, 1) + _vst2_lane_s8(a as _, b.0, b.1, LANE, 1) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43745,7 +43752,7 @@ pub unsafe fn vst2_lane_s16(a: *mut i16, b: int16x4x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4i16")] fn _vst2_lane_s16(ptr: *mut i8, a: int16x4_t, b: int16x4_t, n: i32, size: i32); } - _vst2_s16(a as _, b.0, b.1, LANE, 2) + _vst2_lane_s16(a as _, b.0, b.1, LANE, 2) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43764,7 +43771,7 @@ pub unsafe fn vst2q_lane_s16(a: *mut i16, b: int16x8x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v8i16")] fn _vst2q_lane_s16(ptr: *mut i8, a: int16x8_t, b: int16x8_t, n: i32, size: i32); } - _vst2q_s16(a as _, b.0, b.1, LANE, 2) + _vst2q_lane_s16(a as _, b.0, b.1, LANE, 2) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43783,7 +43790,7 @@ pub unsafe fn vst2_lane_s32(a: *mut i32, b: int32x2x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v2i32")] fn _vst2_lane_s32(ptr: *mut i8, a: int32x2_t, b: int32x2_t, n: i32, size: i32); } - _vst2_s32(a as _, b.0, b.1, LANE, 4) + _vst2_lane_s32(a as _, b.0, b.1, LANE, 4) } #[doc = "Store multiple 2-element structures from two registers"] @@ -43802,7 +43809,7 @@ pub unsafe fn vst2q_lane_s32(a: *mut i32, b: int32x4x2_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst2lane.p0i8.v4i32")] fn _vst2q_lane_s32(ptr: *mut i8, a: int32x4_t, b: int32x4_t, n: i32, size: i32); } - _vst2q_s32(a as _, b.0, b.1, LANE, 4) + _vst2q_lane_s32(a as _, b.0, b.1, LANE, 4) } #[doc = "Store multiple 2-element structures from two registers"] @@ -44657,7 +44664,7 @@ pub unsafe fn vst3_lane_f32(a: *mut f32, b: float32x2x3_t) { size: i32, ); } - _vst3_f32(a as _, b.0, b.1, b.2, LANE, 4) + _vst3_lane_f32(a as _, b.0, b.1, b.2, LANE, 4) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44683,7 +44690,7 @@ pub unsafe fn vst3q_lane_f32(a: *mut f32, b: float32x4x3_t) { size: i32, ); } - _vst3q_f32(a as _, b.0, b.1, b.2, LANE, 4) + _vst3q_lane_f32(a as _, b.0, b.1, b.2, LANE, 4) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44702,7 +44709,7 @@ pub unsafe fn vst3_lane_s8(a: *mut i8, b: int8x8x3_t) { #[cfg_attr(target_arch = "arm", link_name = "llvm.arm.neon.vst3lane.p0i8.v8i8")] fn _vst3_lane_s8(ptr: *mut i8, a: int8x8_t, b: int8x8_t, c: int8x8_t, n: i32, size: i32); } - _vst3_s8(a as _, b.0, b.1, b.2, LANE, 1) + _vst3_lane_s8(a as _, b.0, b.1, b.2, LANE, 1) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44728,7 +44735,7 @@ pub unsafe fn vst3_lane_s16(a: *mut i16, b: int16x4x3_t) { size: i32, ); } - _vst3_s16(a as _, b.0, b.1, b.2, LANE, 2) + _vst3_lane_s16(a as _, b.0, b.1, b.2, LANE, 2) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44754,7 +44761,7 @@ pub unsafe fn vst3q_lane_s16(a: *mut i16, b: int16x8x3_t) { size: i32, ); } - _vst3q_s16(a as _, b.0, b.1, b.2, LANE, 2) + _vst3q_lane_s16(a as _, b.0, b.1, b.2, LANE, 2) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44780,7 +44787,7 @@ pub unsafe fn vst3_lane_s32(a: *mut i32, b: int32x2x3_t) { size: i32, ); } - _vst3_s32(a as _, b.0, b.1, b.2, LANE, 4) + _vst3_lane_s32(a as _, b.0, b.1, b.2, LANE, 4) } #[doc = "Store multiple 3-element structures from three registers"] @@ -44806,7 +44813,7 @@ pub unsafe fn vst3q_lane_s32(a: *mut i32, b: int32x4x3_t) { size: i32, ); } - _vst3q_s32(a as _, b.0, b.1, b.2, LANE, 4) + _vst3q_lane_s32(a as _, b.0, b.1, b.2, LANE, 4) } #[doc = "Store multiple 3-element structures from three registers"] @@ -45865,7 +45872,7 @@ pub unsafe fn vst4_lane_f32(a: *mut f32, b: float32x2x4_t) { size: i32, ); } - _vst4_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vst4_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Store multiple 4-element structures from four registers"] @@ -45892,7 +45899,7 @@ pub unsafe fn vst4q_lane_f32(a: *mut f32, b: float32x4x4_t) { size: i32, ); } - _vst4q_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vst4q_lane_f32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Store multiple 4-element structures from four registers"] @@ -45919,7 +45926,7 @@ pub unsafe fn vst4_lane_s8(a: *mut i8, b: int8x8x4_t) { size: i32, ); } - _vst4_s8(a as _, b.0, b.1, b.2, b.3, LANE, 1) + _vst4_lane_s8(a as _, b.0, b.1, b.2, b.3, LANE, 1) } #[doc = "Store multiple 4-element structures from four registers"] @@ -45946,7 +45953,7 @@ pub unsafe fn vst4_lane_s16(a: *mut i16, b: int16x4x4_t) { size: i32, ); } - _vst4_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) + _vst4_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) } #[doc = "Store multiple 4-element structures from four registers"] @@ -45973,7 +45980,7 @@ pub unsafe fn vst4q_lane_s16(a: *mut i16, b: int16x8x4_t) { size: i32, ); } - _vst4q_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) + _vst4q_lane_s16(a as _, b.0, b.1, b.2, b.3, LANE, 2) } #[doc = "Store multiple 4-element structures from four registers"] @@ -46000,7 +46007,7 @@ pub unsafe fn vst4_lane_s32(a: *mut i32, b: int32x2x4_t) { size: i32, ); } - _vst4_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vst4_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Store multiple 4-element structures from four registers"] @@ -46027,7 +46034,7 @@ pub unsafe fn vst4q_lane_s32(a: *mut i32, b: int32x4x4_t) { size: i32, ); } - _vst4q_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) + _vst4q_lane_s32(a as _, b.0, b.1, b.2, b.3, LANE, 4) } #[doc = "Store multiple 4-element structures from four registers"] diff --git a/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs index 52707f527dc..fe5f8361e02 100644 --- a/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs +++ b/library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs @@ -11,89 +11,33 @@ use crate::{core_arch::simd::*, hint::unreachable_unchecked, intrinsics::simd::* #[cfg(test)] use stdarch_test::assert_instr; -#[unstable(feature = "stdarch_internal", issue = "none")] pub(crate) trait AsUnsigned { type Unsigned: ?Sized; - unsafe fn as_unsigned(self) -> Self::Unsigned; + fn as_unsigned(self) -> Self::Unsigned; } -#[unstable(feature = "stdarch_internal", issue = "none")] pub(crate) trait AsSigned { type Signed: ?Sized; - unsafe fn as_signed(self) -> Self::Signed; -} - -macro_rules! impl_sign_conversions { - ($(($signed:ty, $unsigned:ty))*) => ($( - #[cfg_attr( - not(target_arch = "arm"), - stable(feature = "neon_intrinsics", since = "1.59.0") - )] - #[cfg_attr( - target_arch = "arm", - unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") - )] - impl AsUnsigned for $signed { - type Unsigned = $unsigned; - - #[inline] - unsafe fn as_unsigned(self) -> $unsigned { - crate::mem::transmute(self) - } - } - - #[cfg_attr( - not(target_arch = "arm"), - stable(feature = "neon_intrinsics", since = "1.59.0") - )] - #[cfg_attr( - target_arch = "arm", - unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") - )] - impl AsSigned for $unsigned { - type Signed = $signed; - - #[inline] - unsafe fn as_signed(self) -> $signed { - crate::mem::transmute(self) - } - } - )*) + fn as_signed(self) -> Self::Signed; } macro_rules! impl_sign_conversions_neon { ($(($signed:ty, $unsigned:ty))*) => ($( - #[cfg_attr( - not(target_arch = "arm"), - stable(feature = "neon_intrinsics", since = "1.59.0") - )] - #[cfg_attr( - target_arch = "arm", - unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") - )] impl AsUnsigned for $signed { type Unsigned = $unsigned; #[inline] - unsafe fn as_unsigned(self) -> $unsigned { - crate::mem::transmute(self) + fn as_unsigned(self) -> $unsigned { + unsafe { transmute(self) } } } - #[cfg_attr( - not(target_arch = "arm"), - stable(feature = "neon_intrinsics", since = "1.59.0") - )] - #[cfg_attr( - target_arch = "arm", - unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800") - )] impl AsSigned for $unsigned { type Signed = $signed; #[inline] - unsafe fn as_signed(self) -> $signed { - crate::mem::transmute(self) + fn as_signed(self) -> $signed { + unsafe { transmute(self) } } } )*) @@ -1121,7 +1065,7 @@ pub struct poly64x2x4_t( pub poly64x2_t, ); -impl_sign_conversions! { +impl_sign_conversions_neon! { (i8, u8) (i16, u16) (i32, u32) @@ -1150,9 +1094,6 @@ impl_sign_conversions! { (uint64x2_t, int64x2_t) (uint8x16_t, int8x16_t) (uint8x8_t, int8x8_t) -} - -impl_sign_conversions_neon! { (int16x4x2_t, uint16x4x2_t) (int16x4x3_t, uint16x4x3_t) (int16x4x4_t, uint16x4x4_t) diff --git a/library/stdarch/crates/stdarch-gen-arm/Cargo.toml b/library/stdarch/crates/stdarch-gen-arm/Cargo.toml new file mode 100644 index 00000000000..3cc49825492 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "stdarch-gen-arm" +version = "0.1.0" +authors = ["Luca Vizzarro ", + "Jamie Cunliffe ", + "Adam Gemmell ", + "James Barford-Evans "] +license = "MIT OR Apache-2.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +itertools = "0.10" +lazy_static = "1.4.0" +proc-macro2 = "1.0" +quote = "1.0" +regex = "1.5" +serde = { version = "1.0", features = ["derive"] } +serde_with = "1.14" +serde_yaml = "0.8" +walkdir = "2.3.2" diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml new file mode 100644 index 00000000000..6f622b18be8 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -0,0 +1,9852 @@ +arch_cfgs: + - arch_name: aarch64 + target_feature: [neon] + llvm_prefix: llvm.aarch64.neon + +# Repeatedly used anchors +# #[stable(feature = "neon_intrinsics", since = "1.59.0")] +neon-stable: &neon-stable + FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + +# #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] +neon-unstable: &neon-unstable + FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + +# #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +neon-v7: &neon-v7 + FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v7"']]} ]] + +# #[target_feature(enable = "neon,v7")] +enable-v7: &enable-v7 + FnCall: [target_feature, ['enable = "neon,v7"']] + +# #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +neon-v8: &neon-v8 + FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v8"']]} ]] + +target-is-arm: &target-is-arm + FnCall: [cfg, ['target_arch = "arm"']] + +# #[cfg(not(target_arch = "arm"))] +target-not-arm: &target-not-arm + FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm"']]}]] + +neon-target-aarch64-arm64ec: &neon-target-aarch64-arm64ec + FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]] + +# #[cfg_attr(not(target_arch = "arm"), stable(feature = "neon_intrinsics", since = "1.59.0"))] +neon-stable-not-arm: &neon-stable-not-arm + FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']]}, *neon-stable]] + +#[cfg_attr(target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800"))] +neon-unstable-is-arm: &neon-unstable-is-arm + FnCall: [ cfg_attr, ['target_arch = "arm"', *neon-unstable]] + +# #[cfg_attr(all(test, not(target_env = "msvc"))] +msvc-disabled: &msvc-disabled + FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]] + +# all(test, target_arch = "arm") +test-is-arm: &test-is-arm + FnCall: [all, [test, 'target_arch = "arm"']] + +# #[target_feature(enable = "neon,aes")] +neon-aes: &neon-aes + FnCall: [target_feature, ['enable = "neon,aes"']] + +# #[target_feature(enable = "neon,i8mm")] +neon-i8mm: &neon-i8mm + FnCall: [target_feature, ['enable = "neon,i8mm"']] + +#[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_i8mm", issue = "117223"))] +neon-unstable-i8mm: &neon-unstable-i8mm + FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']] }, { FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] } ]] + +# #[unstable(feature = "stdarch_neon_fcma", issue = "117222")] +neon-unstable-fcma: &neon-unstable-fcma + FnCall: [unstable, ['feature = "stdarch_neon_fcma"', 'issue = "117222"']] + +intrinsics: + - name: "vaddd_{type}" + doc: Add + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - i64 + - u64 + compose: + - MethodCall: + - a + - wrapping_add + - - b + + - name: "veor3{neon_type.no}" + doc: Three-way exclusive OR + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + assert_instr: [eor3] + safety: + unsafe: [neon] + types: + - int8x16_t + - int16x8_t + - int32x4_t + - int64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.crypto.eor3s.{neon_type}" + links: + - link: "llvm.aarch64.crypto.eor3s.{neon_type}" + arch: aarch64,arm64ec + + - name: "veor3{neon_type.no}" + doc: Three-way exclusive OR + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + assert_instr: [eor3] + safety: + unsafe: [neon] + types: + - uint8x16_t + - uint16x8_t + - uint32x4_t + - uint64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.crypto.eor3u.{neon_type}" + links: + - link: "llvm.aarch64.crypto.eor3u.{neon_type}" + arch: aarch64,arm64ec + + - name: "vabd{neon_type.no}" + doc: Absolute difference between the arguments of Floating + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fabd] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fabd.{neon_type}" + links: + - link: "llvm.aarch64.neon.fabd.{neon_type}" + arch: aarch64,arm64ec + + - name: "vabd{type[0]}" + doc: "Floating-point absolute difference" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [fabd] + safety: + unsafe: [neon] + types: + - ['s_f32', 'f32'] + - ['d_f64', 'f64'] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vabd_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[1]}", [b]] + - 0 + + - name: "vabdl_high{neon_type[0].noq}" + doc: Signed Absolute difference Long + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [sabdl] + safety: + unsafe: [neon] + types: + - [int8x16_t, int16x8_t, int8x8_t, uint8x8_t] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - a + - a + - [8, 9, 10, 11, 12, 13, 14, 15] + - Let: + - d + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - b + - b + - [8, 9, 10, 11, 12, 13, 14, 15] + - Let: + - e + - "{neon_type[3]}" + - FnCall: + - simd_cast + - - FnCall: + - "vabd_{neon_type[0]}" + - - c + - d + - FnCall: + - simd_cast + - - e + + - name: "vabdl_high{neon_type[0].noq}" + doc: Signed Absolute difference Long + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + assert_instr: [sabdl] + safety: + unsafe: [neon] + types: + - [int16x8_t, int32x4_t, int16x4_t, uint16x4_t] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - a + - a + - [4, 5, 6, 7] + - Let: + - d + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - b + - b + - [4, 5, 6, 7] + - Let: + - e + - "{neon_type[3]}" + - FnCall: + - simd_cast + - - FnCall: + - "vabd_{neon_type[0]}" + - - c + - d + - FnCall: + - simd_cast + - - e + + - name: "vabdl_high{neon_type[0].noq}" + doc: Signed Absolute difference Long + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + assert_instr: [sabdl] + safety: + unsafe: [neon] + types: + - [int32x4_t, int64x2_t, int32x2_t, uint32x2_t] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - a + - a + - [2, 3] + - Let: + - d + - "{neon_type[2]}" + - FnCall: + - simd_shuffle! + - - b + - b + - [2, 3] + - Let: + - e + - "{neon_type[3]}" + - FnCall: + - simd_cast + - - FnCall: + - "vabd_{neon_type[0]}" + - - c + - d + - FnCall: + - simd_cast + - - e + + - name: "vceq{neon_type[0].no}" + doc: "Compare bitwise Equal (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint64x1_t, uint64x1_t] + - [uint64x2_t, uint64x2_t] + - [int64x1_t, uint64x1_t] + - [int64x2_t, uint64x2_t] + - [poly64x1_t, uint64x1_t] + - [poly64x2_t, uint64x2_t] + compose: + - FnCall: [simd_eq, [a, b]] + + - name: "vceq{neon_type[0].no}" + doc: "Floating-point compare equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmeq]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: [simd_eq, [a, b]] + + - name: "vceq{type[0]}" + doc: "Floating-point compare equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vceq_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[1]}", [b]] + - '0' + + - name: "vceqd_{type[0]}" + doc: "Compare bitwise equal" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64", "s64"] + - ["u64", "u64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - "vceq_{type[2]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vtst{neon_type[0].no}" + doc: "Signed compare bitwise Test bits nonzero" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmtst]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)'] + - [poly64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)'] + - [poly64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)'] + compose: + - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] + - Let: [d, "{type[2]}", "{type[3]}"] + - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] + + - name: "vtstd_{type[0]}" + doc: "Compare bitwise test bits nonzero" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [tst]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64", "s64"] + - ["u64", "u64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - "vtst_{type[2]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vuqadd{type[0]}" + doc: "Signed saturating accumulate of unsigned value" + arguments: ["a: {type[1]}", "b: {type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [suqadd]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_s32", "i32", "u32"] + - ["d_s64", "i64", "u64"] + compose: + - LLVMLink: + name: "vuqadd{type[0]}" + links: + - link: "llvm.aarch64.neon.suqadd.{type[1]}" + arch: aarch64,arm64ec + + - name: "vuqadd{type[0]}" + doc: "Signed saturating accumulate of unsigned value" + arguments: ["a: {type[1]}", "b: {type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [suqadd]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["b_s8", "i8", "u8", "s8"] + - ["h_s16", "i16", "u16", "s16"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vuqadd_{type[3]}" + - - FnCall: ["vdup_n_{type[3]}", [a]] + - FnCall: ["vdup_n_{type[2]}", [b]] + - '0' + + - name: "vabs{neon_type.no}" + doc: "Floating-point absolute value" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fabs]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_fabs, [a]] + + - name: "vcgt{neon_type[0].no}" + doc: "Compare signed greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, uint64x1_t] + - [int64x2_t, uint64x2_t] + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vcgt{neon_type.no}" + doc: "Compare unsigned greater than" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhi]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - uint64x1_t + - uint64x2_t + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vcgt{neon_type[0].no}" + doc: "Floating-point compare greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vcgt{type[0]}" + doc: "Floating-point compare greater than" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - 'simd_extract!' + - - FnCall: + - "vcgt_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[1]}", [b]] + - '0' + + - name: "vclt{neon_type[0].no}" + doc: "Compare signed less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, uint64x1_t] + - [int64x2_t, uint64x2_t] + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vcle{neon_type[0].no}" + doc: "Compare signed less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, uint64x1_t] + - [int64x2_t, uint64x2_t] + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vcle{neon_type[0].no}" + doc: "Floating-point compare less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vcle{type[0]}" + doc: "Floating-point compare less than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vcle_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[1]}", [b]] + - '0' + + - name: "vcge{neon_type[0].no}" + doc: "Compare signed greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, uint64x1_t] + - [int64x2_t, uint64x2_t] + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vcgez{neon_type[0].no}" + doc: "Compare signed greater than or equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: [simd_ge, [a, {FnCall: [transmute, [b]]}]] + + - name: "vcgezd_s64" + doc: "Compare signed greater than or equal to zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - vcgez_s64 + - - FnCall: [transmute, [a]] + + - name: "vclez{neon_type[0].no}" + doc: "Compare signed less than or equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmle]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_le + - - a + - FnCall: [transmute, [b]] + + - name: "vclez{neon_type[0].no}" + doc: "Floating-point compare less than or equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmle]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] + - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] + - [float64x1_t, uint64x1_t, f64, '0.0'] + - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_le + - - a + - FnCall: [transmute, [b]] + + - name: "vclez{type[0]}" + doc: "Floating-point compare less than or equal to zero" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vclez_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - '0' + + - name: "vcltz{neon_type[0].no}" + doc: "Compare signed less than zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmlt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_lt + - - a + - FnCall: [transmute, [b]] + + - name: "vcltz{neon_type[0].no}" + doc: "Floating-point compare less than zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmlt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] + - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] + - [float64x1_t, uint64x1_t, f64, '0.0'] + - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_lt + - - a + - FnCall: [transmute, [b]] + + - name: "vcltz{type[0]}" + doc: "Floating-point compare less than zero" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vcltz_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - '0' + + - name: "vcltzd_s64" + doc: "Compare less than zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [asr]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - vcltz_s64 + - - FnCall: [transmute, [a]] + + - name: "vcagt{neon_type[0].no}" + doc: "Floating-point absolute compare greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcagt{neon_type[0].no}" + links: + - link: "llvm.aarch64.neon.facgt.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcagt{type[0]}" + doc: "Floating-point absolute compare greater than" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32", i32] + - ["d_f64", "f64", "u64", i64] + compose: + - LLVMLink: + name: "vcagt{type[0]}" + links: + - link: "llvm.aarch64.neon.facgt.{type[3]}.{type[1]}" + arch: aarch64,arm64ec + + - name: "vcage{neon_type[0].no}" + doc: "Floating-point absolute compare greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcage{neon_type[0].no}" + links: + - link: "llvm.aarch64.neon.facge.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcage{type[0]}" + doc: "Floating-point absolute compare greater than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32", i32] + - ["d_f64", "f64", "u64", i64] + compose: + - LLVMLink: + name: "vcage{type[0]}" + links: + - link: "llvm.aarch64.neon.facge.{type[3]}.{type[1]}" + arch: aarch64,arm64ec + + - name: "vcalt{neon_type[0].no}" + doc: "Floating-point absolute compare less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: ["vcagt{neon_type[0].no}", [b, a]] + + - name: "vcalt{type[0]}" + doc: "Floating-point absolute compare less than" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: ["vcagt{type[0]}", [b, a]] + + - name: "vcale{neon_type[0].no}" + doc: "Floating-point absolute compare less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: ["vcage{neon_type[0].no}", [b, a]] + + - name: "vcale{type[0]}" + doc: "Floating-point absolute compare less than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: ["vcage{type[0]}", [b, a]] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int64x1_t, float64x1_t] + - [int64x2_t, float64x2_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt{type[0]}_{type[3]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "i32", "f32", s32] + - ["d_f64", "i64", "f64", s64] + compose: + - Identifier: ["a as {type[2]}", Symbol] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint64x1_t, float64x1_t] + - [uint64x2_t, float64x2_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt{type[2]}_{type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u32", "f32", "s_f32"] + - ["u64", "f64", "d_f64"] + compose: + - Identifier: ["a as {type[1]}", Symbol] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x1_t, float64x1_t] + - [int64x2_t, float64x2_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i32", "f32", 's_n_f32_s32', 'N >= 1 && N <= 32'] + - ["i64", "f64", 'd_n_f64_s64', 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: + - "a: {type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxs2fp.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint64x1_t, float64x1_t] + - [uint64x2_t, float64x2_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] + + - name: "vcvt{type[2]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["u32", "f32", 's_n_f32_u32', 'N >= 1 && N <= 32'] + - ["u64", "f64", 'd_n_f64_u64', 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: + - "a: {type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxu2fp.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", ["a.as_signed()", N]] + + - name: "vcvt{type[2]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "i32", "s_s32_f32", "32"] + - ["f64", "i64", "d_s64_f64", "64"] + compose: + - Identifier: ["a as i{type[3]}", Symbol] + + - name: "vcvt{type[2]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", "s_u32_f32"] + - ["f64", "u64", "d_u64_f64"] + compose: + - Identifier: ["a as {type[1]}", Symbol] + + - name: "vcvt_f64_f32" + doc: "Floating-point convert to higher precision long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, float64x2_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt_high_f64_f32" + doc: "Floating-point convert to higher precision long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x4_t, float64x2_t] + compose: + - Let: + - b + - float32x2_t + - FnCall: + - simd_shuffle! + - - a + - a + - '[2, 3]' + - FnCall: [simd_cast, [b]] + + - name: "vcvt_f32_f64" + doc: "Floating-point convert to lower precision narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x2_t, float32x2_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt_high_f32_f64" + doc: "Floating-point convert to lower precision narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, float64x2_t, float32x4_t] + compose: + - FnCall: + - simd_shuffle! + - - a + - FnCall: [simd_cast, [b]] + - '[0, 1, 2, 3]' + + - name: "vcvtx_f32_f64" + doc: "Floating-point convert to lower precision narrow, rounding to odd" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x2_t, float32x2_t] + compose: + - LLVMLink: + name: "vcvtx_f32_f64" + links: + - link: "llvm.aarch64.neon.fcvtxn.v2f32.v2f64" + arch: aarch64,arm64ec + + - name: "vcvtxd_f32_f64" + doc: "Floating-point convert to lower precision narrow, rounding to odd" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f64", "f32"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - vcvtx_f32_f64 + - - FnCall: [vdupq_n_f64, [a]] + - '0' + + - name: "vcvtx_high_f32_f64" + doc: "Floating-point convert to lower precision narrow, rounding to odd" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, float64x2_t, float32x4_t] + compose: + - FnCall: + - simd_shuffle! + - - a + - FnCall: [vcvtx_f32_f64, [b]] + - '[0, 1, 2, 3]' + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float64x1_t, int64x1_t, _n_s64_f64, '64'] + - [float64x2_t, int64x2_t, q_n_s64_f64, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.aarch64.neon.vcvtfp2fxs.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["f32", "i32", s_n_s32_f32, '32'] + - ["f64", "i64", d_n_s64_f64, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.aarch64.neon.vcvtfp2fxs.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t, _n_u64_f64, '64'] + - [float64x2_t, uint64x2_t, q_n_u64_f64, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.aarch64.neon.vcvtfp2fxu.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["f32", "u32", s_n_u32_f32, '32'] + - ["f64", "u64", d_n_u64_f64, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.aarch64.neon.vcvtfp2fxu.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvta{type[2]}" + doc: "Floating-point convert to signed integer, rounding to nearest with ties to away" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtas]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t, _s32_f32] + - [float32x4_t, int32x4_t, q_s32_f32] + - [float64x1_t, int64x1_t, _s64_f64] + - [float64x2_t, int64x2_t, q_s64_f64] + compose: + - LLVMLink: + name: "vcvta{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtas.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvta{type[2]}" + doc: "Floating-point convert to integer, rounding to nearest with ties to away" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtas]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "i32", 's_s32_f32'] + - ["f64", "i64", 'd_s64_f64'] + compose: + - LLVMLink: + name: "vcvta{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtas.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvta{type[2]}" + doc: "Floating-point convert to integer, rounding to nearest with ties to away" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtau]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", 's_u32_f32'] + - ["f64", "u64", 'd_u64_f64'] + compose: + - LLVMLink: + name: "vcvta{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtau.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to signed integer, rounding to nearest with ties to even" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtns]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t] + - [float32x4_t, int32x4_t] + - [float64x1_t, int64x1_t] + - [float64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vcvtn{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtns.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtn{type[2]}" + doc: "Floating-point convert to signed integer, rounding to nearest with ties to even" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtns]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "i32", 's_s32_f32'] + - ["f64", "i64", 'd_s64_f64'] + compose: + - LLVMLink: + name: "vcvtn{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtns.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to signed integer, rounding toward minus infinity" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtms]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t] + - [float32x4_t, int32x4_t] + - [float64x1_t, int64x1_t] + - [float64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vcvtm{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtms.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtm{type[2]}" + doc: "Floating-point convert to signed integer, rounding toward minus infinity" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtms]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "i32", 's_s32_f32'] + - ["f64", "i64", 'd_s64_f64'] + compose: + - LLVMLink: + name: "vcvtm{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtms.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtp{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to signed integer, rounding toward plus infinity" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtps]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t] + - [float32x4_t, int32x4_t] + - [float64x1_t, int64x1_t] + - [float64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vcvtp{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtps.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtp{type[2]}" + doc: "Floating-point convert to signed integer, rounding toward plus infinity" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtps]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "i32", 's_s32_f32'] + - ["f64", "i64", 'd_s64_f64'] + compose: + - LLVMLink: + name: "vcvtp{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtps.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to even" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtnu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcvtn{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtnu.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtn{type[2]}" + doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to even" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtnu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", 's_u32_f32'] + - ["f64", "u64", 'd_u64_f64'] + compose: + - LLVMLink: + name: "vcvtn{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtnu.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned integer, rounding toward minus infinity" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtmu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcvtm{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtmu.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtm{type[2]}" + doc: "Floating-point convert to unsigned integer, rounding toward minus infinity" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtmu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", s_u32_f32] + - ["f64", "u64", d_u64_f64] + compose: + - LLVMLink: + name: "vcvtm{type[2]}" + links: + - link: "llvm.aarch64.neon.fcvtmu.{type[1]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtp{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned integer, rounding toward plus infinity" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtpu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcvtp{neon_type[1].no}_{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.fcvtpu.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvtp{type[2]}" + doc: "Floating-point convert to unsigned integer, rounding toward plus infinity" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtpu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", s_u32_f32, 'i32'] + - ["f64", "u64", d_u64_f64, 'u64'] + compose: + - LLVMLink: + name: "vcvtp{type[2]}" + arguments: + - "a: {type[0]}" + return_type: "{type[1]}" + links: + - link: "llvm.aarch64.neon.fcvtpu.{type[3]}.{type[0]}" + arch: aarch64,arm64ec + + - name: "vdup{neon_type.laneq_nox}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [dup, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - poly64x2_t + - float64x2_t + compose: + - FnCall: [static_assert_uimm_bits!, [N, 1]] + - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] + + - name: "vdup{neon_type[1].lane_nox}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [dup, 'N = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [poly64x1_t, poly64x2_t] + - [float64x1_t, float64x2_t] + compose: + - FnCall: [static_assert!, ['N == 0']] + - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] + + - name: "vdup{neon_type.lane_nox}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - poly64x1_t + - float64x1_t + compose: + - FnCall: [static_assert!, ['N == 0']] + - Identifier: [a, Symbol] + + - name: "vdupd{neon_type[0].lane_nox}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x1_t, "i64"] + - [uint64x1_t, "u64"] + - [float64x1_t, "f64"] + compose: + - FnCall: [static_assert!, ['N == 0']] + - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vdup_laneq_{neon_type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [poly64x2_t, poly64x1_t, 'u64'] + - [float64x2_t, float64x1_t, 'f64'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 1]] + - FnCall: + - "transmute::<{type[2]}, _>" + - - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vdup{type[2]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, "i32", s_lane_s32] + - [int64x2_t, "i64", d_laneq_s64] + - [uint32x2_t, "u32", s_lane_u32] + - [uint64x2_t, "u64", d_laneq_u64] + - [float32x2_t, "f32", s_lane_f32] + - [float64x2_t, "f64", d_laneq_f64] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 1]] + - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vdup{type[2]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 4']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, "i8", b_lane_s8] + - [int16x8_t, "i16", h_laneq_s16] + - [uint8x8_t, "u8", b_lane_u8] + - [uint16x8_t, "u16", h_laneq_u16] + - [poly8x8_t, "p8", b_lane_p8] + - [poly16x8_t, "p16", h_laneq_p16] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 3]] + - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vdup{type[2]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 8']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x16_t, "i8", b_laneq_s8] + - [uint8x16_t, "u8", b_laneq_u8] + - [poly8x16_t, "p8", b_laneq_p8] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 4]] + - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vdup{type[2]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", h_lane_s16] + - [int32x4_t, "i32", s_laneq_s32] + - [uint16x4_t, "u16", h_lane_u16] + - [uint32x4_t, "u32", s_laneq_u32] + - [poly16x4_t, "p16", h_lane_p16] + - [float32x4_t, "f32", s_laneq_f32] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 2]] + - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ext, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [poly64x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + - [float64x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vmla{neon_type.no}" + doc: "Floating-point multiply-add to accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] + + - name: "vmlal_high_{neon_type[1]}" + doc: "Signed multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]', '[4, 5, 6, 7]'] + - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]', '[2, 3]'] + compose: + - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [c, c, "{type[4]}"]]}] + - FnCall: ["vmlal_{neon_type[2]}", [a, b, c]] + + - name: "vmlal_high_{neon_type[1]}" + doc: "Unsigned multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] + compose: + - Let: + - b + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] + - Let: + - c + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + - FnCall: ["vmlal_{neon_type[1]}", [a, b, c]] + + - name: "vmlsl_high_{neon_type[1]}" + doc: "Signed multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]'] + - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]'] + compose: + - Let: + - b + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] + - Let: + - c + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + - FnCall: ["vmlsl_{neon_type[1]}", [a, b, c]] + + - name: "vmlsl_high_{neon_type[1]}" + doc: "Unsigned multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] + compose: + - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [c, c, "{type[3]}"]]}] + - FnCall: ["vmlsl_{neon_type[1]}", [a, b, c]] + + - name: "vmovn_high{neon_type[1].noq}" + doc: Extract narrow + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: [*neon-stable] + assert_instr: [xtn2] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] + - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] + compose: + - Let: + - c + - "{neon_type[0]}" + - FnCall: + - simd_cast + - - b + - FnCall: + - simd_shuffle! + - - a + - c + - "{type[3]}" + + - name: "vneg{neon_type.no}" + doc: Negate + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [neg] + safety: + unsafe: [neon] + types: + - int64x1_t + - int64x2_t + compose: + - FnCall: + - simd_neg + - - a + + - name: "vnegd_s64" + doc: Negate + arguments: ["a: {type}"] + return_type: "{type}" + attr: [*neon-stable] + assert_instr: [neg] + safety: + unsafe: [neon] + types: + - i64 + compose: + - MethodCall: [a, wrapping_neg, []] + + - name: "vneg{neon_type.no}" + doc: Negate + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + assert_instr: [fneg] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: + - simd_neg + - - a + + - name: "vqneg{type[1]}" + doc: Signed saturating negate + arguments: ["a: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqneg] + safety: + unsafe: [neon] + types: + - [i8, 'b_s8', 's8'] + - [i16, 'h_s16', 's16'] + - [i32, 's_s32', 's32'] + - [i64, 'd_s64', 's64'] + compose: + - FnCall: + - 'simd_extract!' + - - FnCall: + - 'vqneg_{type[2]}' + - - FnCall: ['vdup_n_{type[2]}', [a]] + - 0 + + - name: "vqneg{neon_type[0].no}" + doc: Signed saturating negate + arguments: ["a: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqneg] + safety: + unsafe: [neon] + types: + - [int64x1_t, 'i64'] + - [int64x2_t, 'i64'] + compose: + - LLVMLink: + name: "sqneg.{neon_type[0].no}" + links: + - link: "llvm.aarch64.neon.sqneg.v{neon_type[0].lane}{type[1]}" + arch: aarch64,arm64ec + + - name: "vqsub{type[1]}" + doc: Saturating subtract + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqsub] + safety: + unsafe: [neon] + types: + - [i32, 's_s32', 'i32'] + - [i64, 'd_s64', 'i64'] + compose: + - LLVMLink: + name: "sqsub.{type[0]}" + links: + - link: "llvm.aarch64.neon.sqsub.{type[2]}" + arch: aarch64,arm64ec + + - name: "vqsub{type[1]}" + doc: Saturating subtract + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [uqsub] + safety: + unsafe: [neon] + types: + - [u32, 's_u32', 'i32'] + - [u64, 'd_u64', 'i64'] + compose: + - LLVMLink: + name: "uqsub.{type[0]}" + links: + - link: "llvm.aarch64.neon.uqsub.{type[2]}" + arch: aarch64,arm64ec + + - name: "vqsub{type[3]}" + doc: Saturating subtract + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqsub] + safety: + unsafe: [neon] + types: + - [i8, int8x8_t, s8, 'b_s8'] + - [i16, int16x4_t, s16, 'h_s16'] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[2]}" + - - a + - Let: + - b + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[2]}" + - - b + - FnCall: + - 'simd_extract!' + - - FnCall: + - "vqsub_{type[2]}" + - - a + - b + - "0" + + - name: "vqsub{type[3]}" + doc: Saturating subtract + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [uqsub] + safety: + unsafe: [neon] + types: + - [u8, uint8x8_t, u8, 'b_u8'] + - [u16, uint16x4_t, u16, 'h_u16'] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[2]}" + - - a + - Let: + - b + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[2]}" + - - b + - FnCall: + - 'simd_extract!' + - - FnCall: + - "vqsub_{type[2]}" + - - a + - b + - "0" + + - name: "vrbit{neon_type.no}" + doc: Reverse bit order + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [rbit] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + compose: + - LLVMLink: + name: "rbit.{neon_type}" + links: + - link: "llvm.aarch64.neon.rbit.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrbit{neon_type[0].no}" + doc: Reverse bit order + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: [*neon-stable] + assert_instr: [rbit] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [poly8x8_t, int8x8_t] + - [poly8x16_t, int8x16_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vrbit{neon_type[1].no}" + - - FnCall: [transmute, [a]] + + - name: "vrndx{neon_type.no}" + doc: "Floating-point round to integral exact, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frintx] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.rint.{neon_type}" + links: + - link: "llvm.rint.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnda{neon_type.no}" + doc: "Floating-point round to integral, to nearest with ties to away" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frinta] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.round.{neon_type}" + links: + - link: "llvm.round.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrndn{neon_type.no}" + doc: "Floating-point round to integral, to nearest with ties to even" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frintn] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "frintn.{neon_type}" + links: + - link: "llvm.aarch64.neon.frintn.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrndns_{type}" + doc: "Floating-point round to integral, to nearest with ties to even" + arguments: ["a: {type}"] + return_type: "{type}" + attr: [*neon-stable] + assert_instr: [frintn] + safety: + unsafe: [neon] + types: + - f32 + compose: + - LLVMLink: + name: "roundeven.{type}" + links: + - link: "llvm.roundeven.{type}" + arch: aarch64,arm64ec + + - name: "vrndm{neon_type.no}" + doc: "Floating-point round to integral, toward minus infinity" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frintm] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.floor.{neon_type}" + links: + - link: "llvm.floor.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrndp{neon_type.no}" + doc: "Floating-point round to integral, toward plus infinity" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frintp] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.ceil.{neon_type}" + links: + - link: "llvm.ceil.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnd{neon_type.no}" + doc: "Floating-point round to integral, toward zero" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frintz] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.trunc.{neon_type}" + links: + - link: "llvm.trunc.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrndi{neon_type.no}" + doc: "Floating-point round to integral, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [frinti] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.nearbyint.{neon_type}" + links: + - link: "llvm.nearbyint.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqadd{type[1]}" + doc: Saturating add + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [uqadd] + safety: + unsafe: [neon] + types: + - [u32, 's_u32', i32] + - [u64, 'd_u64', i64] + compose: + - LLVMLink: + name: "uqadd.{type[2]}" + links: + - link: "llvm.aarch64.neon.uqadd.{type[2]}" + arch: aarch64,arm64ec + + - name: "vqadd{type[1]}" + doc: Saturating add + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqadd] + safety: + unsafe: [neon] + types: + - [i32, 's_s32', i32] + - [i64, 'd_s64', i64] + compose: + - LLVMLink: + name: "uqadd.{type[2]}" + links: + - link: "llvm.aarch64.neon.sqadd.{type[2]}" + arch: aarch64,arm64ec + + - name: "vqadd{type[2]}" + doc: Saturating add + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [sqadd] + safety: + unsafe: [neon] + types: + - [i8, int8x8_t, 'b_s8'] + - [i16, int16x4_t, 'h_s16'] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[0]}" + - - a + - Let: + - b + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[0]}" + - - b + - FnCall: + - simd_extract! + - - FnCall: + - "vqadd_{type[0]}" + - - a + - b + - "0" + + - name: "vqadd{type[2]}" + doc: Saturating add + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: [*neon-stable] + assert_instr: [uqadd] + safety: + unsafe: [neon] + types: + - [u8, uint8x8_t, 'b_u8'] + - [u16, uint16x4_t, 'h_u16'] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[0]}" + - - a + - Let: + - b + - "{neon_type[1]}" + - FnCall: + - "vdup_n_{type[0]}" + - - b + - FnCall: + - simd_extract! + - - FnCall: + - "vqadd_{type[0]}" + - - a + - b + - "0" + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld1] + safety: + unsafe: [neon] + types: + - ["*const f64", float64x1x2_t] + - ["*const f64", float64x2x2_t] + - ["*const f64", float64x1x3_t] + - ["*const f64", float64x2x3_t] + - ["*const f64", float64x1x4_t] + - ["*const f64", float64x2x4_t] + compose: + - LLVMLink: + name: "vld1{neon_type[1].no}" + links: + - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f64" + arch: aarch64,arm64ec + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const i8", int8x16x2_t, i8, int8x16_t, "4"] + - ["*const i64", int64x2x2_t, i64, int64x2_t, "1"] + - ["*const f64", float64x2x2_t, f64, float64x2_t, "1"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[4]}" + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "a: {neon_type[3]}" + - "b: {neon_type[3]}" + - "n: i64" + - "ptr: *const i8" + links: + - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" + arch: aarch64,arm64ec + - FnCall: ["_vld2{neon_type[1].lane_nox}", ["b.0", "b.1", "LANE as i64", "a as _"]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x2_t, i64, int64x1_t] + - ["*const f64", float64x1x2_t, f64, float64x1_t] + compose: + - FnCall: ["static_assert!", ['LANE == 0']] + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "a: {neon_type[3]}" + - "b: {neon_type[3]}" + - "n: i64" + - "ptr: *const i8" + links: + - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" + arch: aarch64,arm64ec + - FnCall: ["_vld2{neon_type[1].lane_nox}", ["b.0", "b.1", "LANE as i64", "a as _"]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - "static_assert!" + - - 'LANE == 0' + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - "static_assert!" + - - 'LANE == 0' + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x16x2_t, int8x16x2_t, "4"] + - ["*const p8", poly8x16x2_t, int8x16x2_t, "4"] + - ["*const u64", uint64x2x2_t, int64x2x2_t, "1"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[3]}" + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x2_t, int64x2x2_t, "1"] + compose: + - FnCall: ["static_assert_uimm_bits!", [LANE, '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld2] + safety: + unsafe: [neon] + types: + - ["*const f64", float64x2x2_t, f64, float64x2_t] + - ["*const i64", int64x2x2_t, i64, int64x2_t] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const {neon_type[3]}" + links: + - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as _" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["*const f64", float64x1x2_t, f64, float64x1_t] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const {neon_type[3]}" + links: + - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as _" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld2] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-stable + assert_instr: [ld2] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-stable + assert_instr: [ld2r] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x2x2_t, i64] + - ["*const f64", float64x1x2_t, f64] + - ["*const f64", float64x2x2_t, f64] + compose: + - LLVMLink: + name: "vld2dup.{neon_type[1]}" + arguments: + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].dup_nox}" + - - "a as _" + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-stable + assert_instr: [ld2r] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-aes + - *neon-stable + assert_instr: [ld2r] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].lane_nox}" + doc: "Load multiple 3-element structures to two registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i8', int8x16x3_t, int8x16_t, i8, '3'] + - ['*const i64', int64x2x3_t, int64x2_t, i64, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - LLVMLink: + name: 'ld3lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: "Load multiple 3-element structures to three registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const f64', float64x2x3_t, float64x2_t, f64, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - LLVMLink: + name: 'ld3lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const f64', float64x1x3_t, float64x1_t, f64] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'vld3.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: "Load multiple 3-element structures to two registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i64', int64x1x3_t, int64x1_t, i64] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'vld3.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const p8', poly8x16x3_t, int8x16x3_t, '4'] + - ['*const u8', uint8x16x3_t, int8x16x3_t, '4'] + - ['*const u64', uint64x2x3_t, int64x2x3_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const u64', uint64x1x3_t, int64x1x3_t, '1'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x2x3_t, int64x2x3_t] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', 1]] + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x1x3_t, int64x1x3_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + safety: + unsafe: [neon] + assert_instr: [ld3] + types: + - ['*const i64', int64x2x3_t, '*const int64x2_t', i64] + - ['*const f64', float64x2x3_t, '*const float64x2_t', f64] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].nox}' + arguments: + - 'ptr: {type[2]}' + links: + - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + safety: + unsafe: [neon] + assert_instr: [nop] + types: + - ['*const f64', float64x1x3_t, '*const float64x1_t', f64] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].nox}' + arguments: + - 'ptr: {type[2]}' + links: + - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + safety: + unsafe: [neon] + assert_instr: [ld3] + types: + - ['*const u64', uint64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-stable + safety: + unsafe: [neon] + assert_instr: [ld3] + types: + - ['*const p64', poly64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld3r] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x2x3_t, i64] + - ["*const f64", float64x1x3_t, f64] + - ["*const f64", float64x2x3_t, f64] + compose: + - LLVMLink: + name: 'ld3r{neon_type[1].dup_nox}' + arguments: + - 'ptr: {type[0]}' + links: + - link: 'llvm.aarch64.neon.ld3r.v{neon_type[1].lane}{type[2]}.p0{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as _']] + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld3r] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld3{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-stable + assert_instr: [ld3r] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld3{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-stable + assert_instr: [ld4] + safety: + unsafe: [neon] + types: + - ['*const f64', float64x2x4_t, f64, '*const float64x2_t'] + - ['*const i64', int64x2x4_t, i64, '*const int64x2_t'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: {type[3]}' + links: + - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-stable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ['*const f64', float64x1x4_t, f64, '*const float64x1_t'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: {type[3]}' + links: + - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ld4] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-stable + - *neon-aes + assert_instr: [ld4] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i8', int8x16x4_t, int8x16_t, i8, '3'] + - ['*const i64', int64x2x4_t, int64x2_t, i64, '1'] + - ['*const f64', float64x2x4_t, float64x2_t, f64, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - LLVMLink: + name: 'ld4lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i64', int64x1x4_t, int64x1_t, i64] + - ['*const f64', float64x1x4_t, float64x1_t, f64] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'ld4lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const p8', poly8x16x4_t, int8x16x4_t, '4'] + - ['*const u8', uint8x16x4_t, int8x16x4_t, '4'] + - ['*const u64', uint64x2x4_t, int64x2x4_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x2x4_t, int64x2x4_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const u64', uint64x1x4_t, int64x1x4_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: + - target_feature + - - 'enable = "neon,aes"' + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x1x4_t, int64x1x4_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + types: + - ['*mut f64', float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + types: + - ['*mut f64', float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *neon-stable + assert_instr: [st1] + types: + - ['f64', float64x1x2_t, float64x1_t] + compose: + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *neon-stable + assert_instr: [st2] + types: + - [i64, int64x2x2_t, int64x2_t] + - [f64, float64x2x2_t, float64x2_t] + compose: + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [i64, int64x1x2_t, int64x1_t] + - [f64, float64x1x2_t, float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [i8, int8x16x2_t, int8x16_t, '4'] + - [i64, int64x2x2_t, int64x2_t, '1'] + - [f64, float64x2x2_t, float64x2_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [u8, uint8x16x2_t, int8x16x2_t, '4'] + - [u64, uint64x2x2_t, int64x2x2_t, '1'] + - [p8, poly8x16x2_t, int8x16x2_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] + - FnCall: + - "vst2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [u64, uint64x1x2_t, int64x1x2_t, '1'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + assert_instr: [st2] + safety: + unsafe: [neon] + types: + - [u64, uint64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - "vst2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [p64, poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [p64, poly64x2x2_t, int64x2x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '1']] + - FnCall: + - "vst2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2]]}]] + - *neon-stable + safety: + unsafe: [neon] + types: + - [p64, poly64x2x2_t, int64x2x2_t] + compose: + - FnCall: + - "vst2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [f64, float64x1x3_t, float64x1_t] + compose: + - LLVMLink: + name: 'st3.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [p64, poly64x1x3_t, int64x1x3_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst3{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [p64, poly64x2x3_t, int64x2x3_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: + - "vst3{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + assert_instr: [st3] + safety: + unsafe: [neon] + types: + - [p64, poly64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - "vst3{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [st3] + safety: + unsafe: [neon] + types: + - [i64, int64x2x3_t, int64x2_t] + - [f64, float64x2x3_t, float64x2_t] + compose: + - LLVMLink: + name: 'st3.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [st3] + safety: + unsafe: [neon] + types: + - [u64, uint64x2x3_t, int64x2x3_t] + compose: + - FnCall: + - "vst3{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [u64, uint64x1x3_t, int64x1x3_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst3{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [u8, uint8x16x3_t, int8x16x3_t, '4'] + - [u64, uint64x2x3_t, int64x2x3_t, '1'] + - [p8, poly8x16x3_t, int8x16x3_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vst3{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [f64, float64x2x3_t, float64x2_t, '1'] + - [i8, int8x16x3_t, int8x16_t, '4'] + - [i64, int64x2x3_t, int64x2_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - LLVMLink: + name: 'st3lane.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i64, int64x1x3_t, int64x1_t, '1'] + - [f64, float64x1x3_t, float64x1_t, '1'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'st3lane.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [f64, float64x1x4_t, float64x1_t] + compose: + - LLVMLink: + name: 'st4.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [p64, poly64x1x4_t, int64x1x4_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst4{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [p64, poly64x2x4_t, int64x2x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: + - "vst4{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - *neon-aes + assert_instr: [st4] + safety: + unsafe: [neon] + types: + - [p64, poly64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - "vst4{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [st4] + safety: + unsafe: [neon] + types: + - [i64, int64x2x4_t, int64x2_t] + - [f64, float64x2x4_t, float64x2_t] + compose: + - LLVMLink: + name: 'st4.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*neon-stable] + assert_instr: [st4] + safety: + unsafe: [neon] + types: + - [u64, uint64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - "vst4{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [u64, uint64x1x4_t, int64x1x4_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vst4{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-stable + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [u8, uint8x16x4_t, int8x16x4_t, '4'] + - [u64, uint64x2x4_t, int64x2x4_t, '1'] + - [p8, poly8x16x4_t, int8x16x4_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vst4{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [f64, float64x2x4_t, float64x2_t, '1'] + - [i8, int8x16x4_t, int8x16_t, '4'] + - [i64, int64x2x4_t, int64x2_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - LLVMLink: + name: 'st4lane.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i64, int64x1x4_t, int64x1_t, '1'] + - [f64, float64x1x4_t, float64x1_t, '1'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - LLVMLink: + name: 'st4lane.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vusdot{neon_type[0].laneq_nox}" + doc: "Dot product index form with unsigned and signed integers" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-i8mm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usdot, 'LANE = 3']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [int32x2_t, uint8x8_t, int8x16_t, '[LANE as u32, LANE as u32]'] + - [int32x4_t, uint8x16_t, int8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: [c, int32x4_t, {FnCall: [transmute, [c]]}] + - Let: [c, "{neon_type[0]}", {FnCall: [simd_shuffle!, [c, c, "{type[3]}"]]}] + - FnCall: ["vusdot{neon_type[0].no}", [a, b, {FnCall: [transmute, [c]]}]] + + - name: "vsudot{neon_type[0].laneq_nox}" + doc: "Dot product index form with signed and unsigned integers" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-i8mm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sudot, 'LANE = 3']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [int32x2_t, int8x8_t, uint8x16_t, '[LANE as u32, LANE as u32]', uint32x2_t] + - [int32x4_t, int8x16_t, uint8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]', uint32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 2]] + - Let: + - c + - uint32x4_t + - FnCall: [transmute, [c]] + - Let: + - c + - "{type[4]}" + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + - FnCall: ["vusdot{neon_type[0].no}", [a, {FnCall: [transmute, [c]]}, b]] + + - name: "vmul{neon_type.no}" + doc: Multiply + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fmul] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_mul, [a, b]] + + - name: "vmull_high{neon_type[0].noq}" + doc: Signed multiply long + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[3]}" + attr: [*neon-stable] + assert_instr: [smull2] + safety: + unsafe: [neon] + types: + - [int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int16x8_t] + - [int16x8_t, int16x4_t, '[4, 5, 6, 7]', int32x4_t] + - [int32x4_t, int32x2_t, '[2, 3]', int64x2_t] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] + - Let: + - b + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] + - FnCall: ["vmull_{neon_type[0]}", [a, b]] + + - name: "vmull_high{neon_type[0].noq}" + doc: "Unsigned multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[3]}" + attr: [*neon-stable] + assert_instr: [umull2] + safety: + unsafe: [neon] + types: + - [uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', uint16x8_t] + - [uint16x8_t, uint16x4_t, '[4, 5, 6, 7]', uint32x4_t] + - [uint32x4_t, uint32x2_t, '[2, 3]', uint64x2_t] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] + - Let: + - b + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] + - FnCall: ["vmull_{neon_type[0]}", [a, b]] + + - name: "vmull_p64" + doc: "Polynomial multiply long" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-aes + - *neon-stable + safety: + unsafe: [neon] + assert_instr: [pmull] + types: + - ["p64", "p128"] + compose: + - LLVMLink: + name: "pmull.{type[0]}" + return_type: "int8x16_t" + links: + - link: "llvm.aarch64.neon.pmull64" + arch: aarch64,arm64ec + - FnCall: [transmute, [{FnCall: ["_vmull_p64", [a, b]]}]] + + - name: "vmull_high{neon_type[0].noq}" + doc: "Polynomial multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[3]}" + attr: + - *neon-stable + safety: + unsafe: [neon] + assert_instr: [pmull] + types: + - [poly8x16_t, poly8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', poly16x8_t] + compose: + - Let: + - a + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] + - Let: + - b + - "{neon_type[1]}" + - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] + - FnCall: ["vmull_{neon_type[0]}", [a, b]] + + - name: "vmull_high{neon_type[0].noq}" + doc: "Polynomial multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-aes + - *neon-stable + safety: + unsafe: [neon] + assert_instr: [pmull] + types: + - [poly64x2_t, "p128"] + compose: + - FnCall: + - "vmull_{neon_type[0]}" + - - FnCall: [simd_extract!, [a, '1']] + - FnCall: [simd_extract!, [b, '1']] + + - name: "vmulx{neon_type.no}" + doc: Floating-point multiply extended + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fmulx] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fmulx.{neon_type.no}" + links: + - link: "llvm.aarch64.neon.fmulx.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmulx{type[0]}" + doc: Floating-point multiply extended + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [fmulx] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32"] + - ["d_f64", "f64"] + compose: + - LLVMLink: + name: "fmulx.{type[1]}" + links: + - link: "llvm.aarch64.neon.fmulx.{type[1]}" + arch: aarch64,arm64ec + + - name: "vmulx_lane_f64" + doc: Floating-point multiply extended + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - vmulx_f64 + - - a + - FnCall: + - 'transmute::' + - - FnCall: + - "simd_extract!" + - - b + - 'LANE as u32' + + - name: "vmulx{type[0]}" + doc: Floating-point multiply extended + arguments: ["a: {type[1]}", "b: {neon_type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ["q_lane_f64", float64x2_t, float64x1_t, "q_f64", '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vmulx{type[3]}" + - - a + - FnCall: + - "simd_shuffle!" + - - b + - b + - "{type[4]}" + + - name: "vmulx{type[0]}" + doc: Floating-point multiply extended + arguments: ["a: {type[1]}", "b: {neon_type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ["d_lane_f64", "f64", float64x1_t, "d_f64", 'LANE as u32'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - "vmulx{type[3]}" + - - a + - FnCall: + - "simd_extract!" + - - b + - "{type[4]}" + + - name: "vmulx_laneq_f64" + doc: Floating-point multiply extended + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float64x1_t, float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '1']] + - FnCall: + - vmulx_f64 + - - a + - FnCall: + - 'transmute::' + - - FnCall: + - "simd_extract!" + - - b + - 'LANE as u32' + + - name: "vmulx{type[0]}" + doc: Floating-point multiply extended + arguments: ["a: {type[1]}", "b: {neon_type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['_lane_f32', float32x2_t, float32x2_t, '1', '_f32', '[LANE as u32, LANE as u32]'] + - ['_laneq_f32', float32x2_t, float32x4_t, '2', '_f32', '[LANE as u32, LANE as u32]'] + - ['q_lane_f32', float32x4_t, float32x2_t, '1', 'q_f32', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - ['q_laneq_f32', float32x4_t, float32x4_t, '2', 'q_f32', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - ['q_laneq_f64', float64x2_t, float64x2_t, '1', 'q_f64', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] + - FnCall: + - "vmulx{type[4]}" + - - a + - FnCall: + - "simd_shuffle!" + - - b + - b + - "{type[5]}" + + - name: "vmulx{type[0]}" + doc: Floating-point multiply extended + arguments: ["a: {type[1]}", "b: {neon_type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['s_lane_f32', f32, float32x2_t, '1', 's_f32', 'LANE as u32'] + - ['s_laneq_f32', f32, float32x4_t, '2', 's_f32', 'LANE as u32'] + - ['d_laneq_f64', f64, float64x2_t, '1', 'd_f64', 'LANE as u32'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] + - FnCall: + - "vmulx{type[4]}" + - - a + - FnCall: + - "simd_extract!" + - - b + - "{type[5]}" + + - name: "vfma{neon_type.no}" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-stable + assert_instr: [fmadd] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - LLVMLink: + name: "_vfma{neon_type.no}" + arguments: ["a: {type}", "b: {type}", "c: {type}"] + links: + - link: "llvm.fma.{neon_type}" + arch: aarch64,arm64ec + - FnCall: ["_vfma{neon_type.no}", [b, c, a]] + + - name: "vfma{neon_type.no}" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-stable + assert_instr: [fmla] + safety: + unsafe: [neon] + types: + - float64x2_t + compose: + - LLVMLink: + name: "_vfma{neon_type.no}" + arguments: ["a: {type}", "b: {type}", "c: {type}"] + links: + - link: "llvm.fma.{neon_type}" + arch: aarch64,arm64ec + - FnCall: ["_vfma{neon_type.no}", [b, c, a]] + + - name: "vfma_n_f64" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + assert_instr: [fmadd] + safety: + unsafe: [neon] + types: + - [float64x1_t, f64] + compose: + - FnCall: + - "vfma_f64" + - - a + - b + - FnCall: + - "vdup_n_f64" + - - c + + - name: "vfmaq_n_f64" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + assert_instr: [fmla] + safety: + unsafe: [neon] + types: + - [float64x2_t, f64] + compose: + - FnCall: + - "vfmaq_f64" + - - a + - b + - FnCall: + - "vdupq_n_f64" + - - c + + - name: "vdiv{neon_type.no}" + doc: "Divide" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fdiv] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_div, [a, b]] + + - name: "vsub{neon_type.no}" + doc: "Subtract" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fsub] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_sub, [a, b]] + + - name: "vsub{type[0]}" + doc: "Subtract" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ['d_s64', 'i64'] + - ['d_u64', 'u64'] + compose: + - MethodCall: [a, wrapping_sub, [b]] + + - name: "vaddv{neon_type[0].no}" + doc: Floating-point add across vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + assert_instr: [faddp] + safety: + unsafe: [neon] + types: + - [float32x2_t, f32] + - [float32x4_t, f32] + - [float64x2_t, f64] + compose: + - LLVMLink: + name: "faddv.{type[1]}.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.faddv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vaddlv{neon_type[0].no}" + doc: Signed Add Long across Vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [saddlv] + safety: + unsafe: [neon] + types: + - [int16x4_t, i32] + - [int16x8_t, i32] + - [int32x4_t, i64] + compose: + - LLVMLink: + name: "llvm.aarch64.neon.saddlv.{type[1]}.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.saddlv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vaddlv{neon_type.no}" + doc: Signed Add Long across Vector + arguments: ["a: {neon_type}"] + return_type: "i64" + attr: [*neon-stable] + assert_instr: [saddlp] + safety: + unsafe: [neon] + types: + - int32x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.saddlv.i64.v2i32" + links: + - link: "llvm.aarch64.neon.saddlv.i64.v2i32" + arch: aarch64,arm64ec + + - name: "vaddlv{neon_type[0].no}" + doc: Unsigned Add Long across Vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [uaddlv] + safety: + unsafe: [neon] + types: + - [uint16x4_t, u32, i32] + - [uint16x8_t, u32, i32] + - [uint32x4_t, u64, i64] + compose: + - LLVMLink: + name: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vaddlv{neon_type[0].no}" + doc: Unsigned Add Long across Vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [uaddlp] + safety: + unsafe: [neon] + types: + - [uint32x2_t, u64, i64] + compose: + - LLVMLink: + name: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vsubw_high{neon_type[1].noq}" + doc: Signed Subtract Wide + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: [*neon-stable] + assert_instr: [ssubw] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]'] + - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]'] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] + - FnCall: + - simd_sub + - - a + - FnCall: [simd_cast, [c]] + + - name: "vsubw_high{neon_type[1].noq}" + doc: Unsigned Subtract Wide + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: [*neon-stable] + assert_instr: [usubw] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] + - FnCall: + - simd_sub + - - a + - FnCall: [simd_cast, [c]] + + - name: "vsubl_high{neon_type[0].noq}" + doc: "Signed Subtract Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [ssubl] + safety: + unsafe: [neon] + types: + - [int8x16_t, int16x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int8x8_t] + - [int16x8_t, int32x4_t, '[4, 5, 6, 7]', int16x4_t] + - [int32x4_t, int64x2_t, '[2, 3]', int32x2_t] + compose: + - Let: + - c + - "{neon_type[3]}" + - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] + - Let: + - d + - "{neon_type[1]}" + - FnCall: [simd_cast, [c]] + - Let: + - e + - "{neon_type[3]}" + - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] + - Let: + - f + - "{neon_type[1]}" + - FnCall: [simd_cast, [e]] + - FnCall: [simd_sub, [d, f]] + + - name: "vsubl_high{neon_type[0].noq}" + doc: "Unsigned Subtract Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [usubl] + safety: + unsafe: [neon] + types: + - [uint8x16_t, uint16x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', uint8x8_t] + - [uint16x8_t, uint32x4_t, '[4, 5, 6, 7]', uint16x4_t] + - [uint32x4_t, uint64x2_t, '[2, 3]', uint32x2_t] + compose: + - Let: + - c + - "{neon_type[3]}" + - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] + - Let: + - d + - "{neon_type[1]}" + - FnCall: [simd_cast, [c]] + - Let: + - e + - "{neon_type[3]}" + - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] + - Let: + - f + - "{neon_type[1]}" + - FnCall: [simd_cast, [e]] + - FnCall: [simd_sub, [d, f]] + + - name: "vbcax{neon_type.no}" + doc: Bit clear and exclusive OR + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + assert_instr: [bcax] + safety: + unsafe: [neon] + types: + - int8x16_t + - int16x8_t + - int32x4_t + - int64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.crypto.bcaxs.{neon_type}" + links: + - link: "llvm.aarch64.crypto.bcaxs.{neon_type}" + arch: aarch64,arm64ec + + - name: "vbcax{neon_type.no}" + doc: Bit clear and exclusive OR + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + assert_instr: [bcax] + safety: + unsafe: [neon] + types: + - uint8x16_t + - uint16x8_t + - uint32x4_t + - uint64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.crypto.bcaxu.{neon_type}" + links: + - link: "llvm.aarch64.crypto.bcaxu.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcadd{neon_type.rot270}" + doc: "Floating-point complex add" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcadd] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcadd.rot270.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcadd.rot270.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcadd{neon_type.rot90}" + doc: "Floating-point complex add" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcadd] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcadd.rot90.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcadd.rot90.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcmla{neon_type.no}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcmla] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcmla.rot0.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcmla.rot0.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcmla{neon_type.rot90}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcmla] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcmla.rot90.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcmla.rot90.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcmla{neon_type.rot270}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcmla] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcmla.rot270.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcmla.rot270.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcmla{neon_type[0].laneq_nox}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 1]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].no}", [a, b, c]] + + - name: "vcmla{neon_type[0].rot90_laneq}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 1]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].rot90}", [a, b, c]] + + - name: "vcmla{neon_type[0].rot90_lane}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].rot90}", [a, b, c]] + + - name: "vcmla{neon_type.rot180}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - *neon-unstable-fcma + assert_instr: [fcmla] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "llvm.aarch64.neon.vcmla.rot180.{neon_type}" + links: + - link: "llvm.aarch64.neon.vcmla.rot180.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcmla{neon_type[0].rot180_laneq}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 1]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].rot180}", [a, b, c]] + + - name: "vcmla{type[3]}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]', '_rot180_lane_f32'] + - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]', 'q_rot180_lane_f32'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].rot180}", [a, b, c]] + + - name: "vcmla{neon_type[0].rot270_laneq}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, 1]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].rot270}", [a, b, c]] + + - name: "vcmla{neon_type[0].lane_nox}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] + - FnCall: ["vcmla{neon_type[0].no}", [a, b, c]] + + - name: "vcmla{neon_type[0].rot270_lane}" + doc: Floating-point complex multiply accumulate + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [target_feature, ['enable = "neon,fcma"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-fcma + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] + - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Let: [c, "{neon_type[0]}", {FnCall: [simd_shuffle!, [c, c, "{type[2]}"]]}] + - FnCall: ["vcmla{neon_type[0].rot270}", [a, b, c]] + + - name: "vdot{neon_type[0].laneq_nox}" + doc: Dot product arithmetic (indexed) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + static_defs: ["const LANE: i32"] + attr: + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sdot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']] + safety: + unsafe: [neon] + types: + - [int32x2_t, int8x8_t, int8x16_t, int32x4_t, '[LANE as u32, LANE as u32]'] + - [int32x4_t, int8x16_t, int8x16_t, int32x4_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: + - c + - "{neon_type[3]}" + - FnCall: [transmute, [c]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] + - FnCall: + - "vdot{neon_type[0].no}" + - - a + - b + - FnCall: [transmute, [c]] + + - name: "vdot{neon_type[0].laneq_nox}" + doc: Dot product arithmetic (indexed) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + static_defs: ["const LANE: i32"] + attr: + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [udot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']] + safety: + unsafe: [neon] + types: + - [uint32x2_t, uint8x8_t, uint8x16_t, uint32x4_t, '[LANE as u32, LANE as u32]'] + - [uint32x4_t, uint8x16_t, uint8x16_t, uint32x4_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '2']] + - Let: + - c + - "{neon_type[3]}" + - FnCall: [transmute, [c]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] + - FnCall: + - "vdot{neon_type[0].no}" + - - a + - b + - FnCall: [transmute, [c]] + + - name: "vmax{neon_type.no}" + doc: Maximum (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fmax] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fmax.{neon_type}" + links: + - link: "llvm.aarch64.neon.fmax.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmaxnm{neon_type.no}" + doc: Floating-point Maximum Number (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fmaxnm] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fmaxnm.{neon_type}" + links: + - link: "llvm.aarch64.neon.fmaxnm.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmaxnmv{neon_type[0].no}" + doc: Floating-point maximum number across vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [fmaxnmp] + safety: + unsafe: [neon] + types: + - [float32x2_t, f32] + - [float64x2_t, f64] + compose: + - LLVMLink: + name: "fmaxnmv.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fmaxnmv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vmaxnmv{neon_type[0].no}" + doc: Floating-point maximum number across vector + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [fmaxnmv] + safety: + unsafe: [neon] + types: + - [float32x4_t, f32] + compose: + - LLVMLink: + name: "fmaxnmv.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fmaxnmv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vpmax{type[0]}" + doc: "Floating-point maximum pairwise" + arguments: ["a: {neon_type[1]}"] + return_type: "{type[2]}" + attr: [*neon-stable] + assert_instr: [fmaxp] + safety: + unsafe: [neon] + types: + - ["s_f32", float32x2_t, f32] + - ["qd_f64", float64x2_t, f64] + compose: + - LLVMLink: + name: "fmaxv.{type[0]}" + links: + - link: "llvm.aarch64.neon.fmaxv.{type[2]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vmin{neon_type.no}" + doc: "Minimum (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fmin] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fmin.{neon_type}" + links: + - link: "llvm.aarch64.neon.fmin.{neon_type}" + arch: aarch64,arm64ec + + - name: "vminnm{neon_type.no}" + doc: "Floating-point Minimum Number (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: [*neon-stable] + assert_instr: [fminnm] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "fminnm.{neon_type}" + links: + - link: "llvm.aarch64.neon.fminnm.{neon_type}" + arch: aarch64,arm64ec + + - name: "vminnmv{neon_type[0].no}" + doc: "Floating-point minimum number across vector" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, "f32"] + - [float64x2_t, "f64"] + compose: + - LLVMLink: + name: "vminnmv.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fminnmv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vminnmv{neon_type[0].no}" + doc: "Floating-point minimum number across vector" + arguments: ["a: {neon_type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmv]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x4_t, "f32"] + compose: + - LLVMLink: + name: "vminnmv.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fminnmv.{type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vmovl_high{neon_type[0].noq}" + doc: Vector move + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [sxtl2] + safety: + unsafe: [neon] + types: + - [int8x16_t, int16x8_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]'] + - [int32x4_t, int64x2_t, int32x2_t, '[2, 3]'] + compose: + - Let: + - a + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] + - FnCall: ["vmovl{neon_type[0].noq}", [a]] + + - name: "vmovl_high{neon_type[0].noq}" + doc: Vector move + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: [*neon-stable] + assert_instr: [uxtl2] + safety: + unsafe: [neon] + types: + - [uint8x16_t, uint16x8_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x8_t, uint32x4_t, uint16x4_t, '[4, 5, 6, 7]'] + - [uint32x4_t, uint64x2_t, uint32x2_t, '[2, 3]'] + compose: + - Let: + - a + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] + - FnCall: ["vmovl{neon_type[0].noq}", [a]] + + - name: "vpadd{neon_type.no}" + doc: Floating-point add pairwise + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{type}" + attr: [*neon-stable] + assert_instr: [faddp] + safety: + unsafe: [neon] + types: + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "faddp.{neon_type}" + links: + - link: "llvm.aarch64.neon.faddp.{neon_type}" + arch: aarch64,arm64ec + + - name: "vpadd{type[0]}" + doc: "Floating-point add pairwise" + arguments: ["a: {neon_type[1]}"] + return_type: "{type[2]}" + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["s_f32", float32x2_t, f32] + - ["d_f64", float64x2_t, f64] + compose: + - Let: + - a1 + - "{type[2]}" + - FnCall: [simd_extract!, [a, '0']] + - Let: + - a2 + - "{type[2]}" + - FnCall: [simd_extract!, [a, '1']] + - Identifier: ['a1 + a2', Symbol] + + - name: "vpmin{type[0]}" + doc: Floating-point minimum pairwise + arguments: ["a: {neon_type[1]}"] + return_type: "{type[2]}" + attr: [*neon-stable] + assert_instr: [fminp] + safety: + unsafe: [neon] + types: + - ["s_f32", float32x2_t, f32] + - ["qd_f64", float64x2_t, f64] + compose: + - LLVMLink: + name: "fminv.{type[2]}.{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.fminv.{type[2]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqdmullh_s16" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i16", "i32"] + compose: + - Let: [a, int16x4_t, {FnCall: [vdup_n_s16, [a]]}] + - Let: [b, int16x4_t, {FnCall: [vdup_n_s16, [b]]}] + - FnCall: [simd_extract!, [{FnCall: [vqdmull_s16, [a, b]]}, '0']] + + - name: "vqdmulls_s32" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i32", "i64"] + compose: + - LLVMLink: + name: "vqdmulls_s32" + links: + - link: "llvm.aarch64.neon.sqdmulls.scalar" + arch: aarch64,arm64ec + + - name: "vqdmull_high{neon_type[0].noq}" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]'] + - [int32x4_t, int64x2_t, int32x2_t, '[2, 3]'] + compose: + - Let: [a, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, '{type[3]}']]}] + - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, '{type[3]}']]}] + - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] + + - name: "vqdmull_high_n_{type[1]}" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int16x8_t, "i16", int32x4_t, int16x4_t, '[4, 5, 6, 7]'] + - [int32x4_t, "i32", int64x2_t, int32x2_t, '[2, 3]'] + compose: + - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - Let: [b, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[0].noq}", [b]]}] + - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] + + - name: "vqdmull{type[3]}" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, "i32", 'h_lane_s16', 'h_s16'] + - ["i32", int32x4_t, "i64", 's_laneq_s32', 's_s32'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 2]] + - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] + - FnCall: ["vqdmull{type[4]}", [a, b]] + + - name: "vqdmullh_laneq_s16" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, N = 4]]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x8_t, "i32"] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 3]] + - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] + - FnCall: ["vqdmullh_s16", [a, b]] + + - name: "vqdmulls_lane_s32" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i32", int32x2_t, "i64"] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 1]] + - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] + - FnCall: ["vqdmulls_s32", [a, b]] + + - name: "vqdmull{type[6]}" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int16x4_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]', '[N as u32, N as u32, N as u32, N as u32]', '_high_lane_s16'] + - [int32x4_t, int32x4_t, int64x2_t, int32x2_t, '[2, 3]', '[N as u32, N as u32]', '_high_laneq_s32'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] + - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] + + - name: "vqdmull_high_lane_s32" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int32x2_t, int64x2_t, int32x2_t, '[2, 3]', '[N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '1']] + - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] + - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] + + - name: "vqdmull_high_laneq_s16" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, N = 4]]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]', '[N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '3']] + - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] + - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] + + - name: "vqdmull_laneq_s16" + doc: "Vector saturating doubling long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 4']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x8_t, int32x4_t, '[N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '3']] + - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - FnCall: [vqdmull_s16, [a, b]] + + - name: "vqdmull_laneq_s32" + doc: "Vector saturating doubling long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, int32x4_t, int64x2_t, '[N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - FnCall: [vqdmull_s32, [a, b]] + + - name: "vqdmlal{type[4]}" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_s16] + - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_s32] + - [int32x4_t, int16x8_t, "i16", int32x4_t, _high_n_s16] + - [int64x2_t, int32x4_t, "i32", int64x2_t, _high_n_s32] + compose: + - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] + + - name: "vqdmlal{type[4]}" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal2, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x4_t, int32x4_t, _high_lane_s16, '2'] + - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_laneq_s16, '3'] + - [int64x2_t, int32x4_t, int32x2_t, int64x2_t, _high_lane_s32, '1'] + - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_laneq_s32, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[5]}"]] + - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}::", [b, c]]}]] + + - name: "vqdmlalh_{type[2]}" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i32", "i16", "s16"] + compose: + - Let: [x, int32x4_t, {FnCall: [vqdmull_s16, [{FnCall: [vdup_n_s16, [b]]}, {FnCall: [vdup_n_s16, [c]]}]]}] + - FnCall: [vqadds_s32, [a, {FnCall: [simd_extract!, [x, 0]]}]] + + - name: "vqdmlals_s32" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "i32", "i32", "i64"] + compose: + - Let: [x, i64, {FnCall: [vqaddd_s64, [a, {FnCall: [vqdmulls_s32, [b, c]]}]]}] + - Identifier: ['x as i64', Symbol] + + - name: "vqdmlal{type[4]}" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {neon_type[2]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i32", "i16", int16x4_t, "i32", h_lane_s16, '2', h_s16] + - ["i32", "i16", int16x8_t, "i32", h_laneq_s16, '3', h_s16] + - ["i64", "i32", int32x2_t, "i64", s_lane_s32, '1', s_s32] + - ["i64", "i32", int32x4_t, "i64", s_laneq_s32, '2', s_s32] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[5]}"]] + - FnCall: ["vqdmlal{type[6]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vqdmlal_laneq_s16" + doc: "Vector widening saturating doubling multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x8_t, int32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '3']] + - FnCall: [vqaddq_s32, [a, {FnCall: ["vqdmull_laneq_s16::", [b, c]]}]] + + - name: "vqdmlal_laneq_s32" + doc: "Vector widening saturating doubling multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int32x2_t, int32x4_t, int64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - FnCall: [vqaddq_s64, [a, {FnCall: ["vqdmull_laneq_s32::", [b, c]]}]] + + - name: "vqdmlsl{type[4]}" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_s16] + - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_s32] + - [int32x4_t, int16x8_t, "i16", int32x4_t, _high_n_s16] + - [int64x2_t, int32x4_t, "i32", int64x2_t, _high_n_s32] + compose: + - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] + + - name: "vqdmlsl{type[4]}" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl2, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x4_t, int32x4_t, '_high_lane_s16', '2'] + - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, '_high_laneq_s16', '3'] + - [int64x2_t, int32x4_t, int32x2_t, int64x2_t, '_high_lane_s32', '1'] + - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, '_high_laneq_s32', '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[5]}"]] + - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}::", [b, c]]}]] + + - name: "vqdmlslh_s16" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i32", "i16"] + compose: + - Let: [x, int32x4_t, {FnCall: [vqdmull_s16, [{FnCall: [vdup_n_s16, [b]]}, {FnCall: [vdup_n_s16, [c]]}]]}] + - FnCall: [vqsubs_s32, [a, {FnCall: [simd_extract!, [x, '0']]}]] + + - name: "vqdmlsls_s32" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "i32", "i32", "i64"] + compose: + - Let: [x, i64, {FnCall: [vqsubd_s64, [a, {FnCall: [vqdmulls_s32, [b, c]]}]]}] + - Identifier: ['x as i64', Symbol] + + - name: "vqdmlsl{type[4]}" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {type[0]}", "b: {type[1]}", "c: {neon_type[2]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i32", "i16", int16x4_t, "i32", 'h_lane_s16', '2', 'h_s16'] + - ["i32", "i16", int16x8_t, "i32", 'h_laneq_s16', '3', 'h_s16'] + - ["i64", "i32", int32x2_t, "i64", 's_lane_s32', '1', 's_s32'] + - ["i64", "i32", int32x4_t, "i64", 's_laneq_s32', '2', 's_s32'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[5]}"]] + - FnCall: ["vqdmlsl{type[6]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vqdmlsl_laneq_s16" + doc: "Vector widening saturating doubling multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x8_t, int32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '3']] + - FnCall: ["vqsubq_s32", [a, {FnCall: ["vqdmull_laneq_s16::", [b, c]]}]] + + - name: "vqdmlsl_laneq_s32" + doc: "Vector widening saturating doubling multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int32x2_t, int32x4_t, int64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - FnCall: [vqsubq_s64, [a, {FnCall: ["vqdmull_laneq_s32::", [b, c]]}]] + + - name: "vqdmulh{type[4]}" + doc: "Signed saturating doubling multiply returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i16", "i16", "i16", int16x4_t, 'h_s16'] + - ["i32", "i32", "i32", int32x2_t, 's_s32'] + compose: + - Let: [a, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[3].no}", [a]]}] + - Let: [b, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[3].no}", [b]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqdmulh{neon_type[3].no}", [a, b]]}, '0']] + + - name: "vqdmulhh{type[3]}" + doc: "Signed saturating doubling multiply returning high half" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, "i16", '_lane_s16', '2'] + - ["i16", int16x8_t, "i16", '_laneq_s16', '3'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[4]}"]] + - Let: [b, 'i16', {FnCall: [simd_extract!, [b, 'N as u32']]}] + - FnCall: ['vqdmulhh_s16', [a, b]] + + - name: "vqdmulhs{type[3]}" + doc: "Signed saturating doubling multiply returning high half" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i32", int32x2_t, "i32", "_lane_s32", '1'] + - ["i32", int32x4_t, "i32", "_laneq_s32", '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[4]}"]] + - Let: [b, 'i32', {FnCall: [simd_extract!, [b, 'N as u32']]}] + - FnCall: ['vqdmulhs_s32', [a, b]] + + - name: "vqmovn_high{neon_type[1].noq}" + doc: "Signed saturating extract narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] + compose: + - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovn{neon_type[1].noq}", [b]]}, "{type[3]}"]] + + - name: "vqmovn_high{neon_type[1].noq}" + doc: "Signed saturating extract narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] + compose: + - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovn{neon_type[1].noq}", [b]]}, "{type[3]}"]] + + - name: "vqmovn{type[2]}" + doc: "Saturating extract narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i16", "i8", 'h_s16', s16] + - ["i32", "i16", 's_s32', s32] + compose: + - FnCall: [simd_extract!, [{FnCall: ["vqmovn_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] + + - name: "vqmovn{type[2]}" + doc: "Saturating extract narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u16", "u8", 'h_u16', 'u16'] + - ["u32", "u16", 's_u32', 'u32'] + compose: + - FnCall: [simd_extract!, [{FnCall: ["vqmovn_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] + + - name: "vqmovnd_s64" + doc: "Saturating extract narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "i32"] + compose: + - LLVMLink: + name: "vqmovnd_s64" + links: + - link: "llvm.aarch64.neon.scalar.sqxtn.i32.i64" + arch: aarch64,arm64ec + + - name: "vqmovnd_u64" + doc: "Saturating extract narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u64", "u32"] + compose: + - LLVMLink: + name: "vqmovnd_u64" + links: + - link: "llvm.aarch64.neon.scalar.uqxtn.i32.i64" + arch: aarch64,arm64ec + + - name: "vqmovun{type[2]}" + doc: "Signed saturating extract unsigned narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtun]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i16", "u8", 'h_s16', s16] + - ["i32", "u16", 's_s32', s32] + - ["i64", "u32", 'd_s64', s64] + compose: + - FnCall: [simd_extract!, [{FnCall: ["vqmovun_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] + + - name: "vqmovun_high_{neon_type[1]}" + doc: "Signed saturating extract unsigned narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtun2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int16x8_t, uint8x16_t, s16, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, int32x4_t, uint16x8_t, s32, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, int64x2_t, uint32x4_t, s64, '[0, 1, 2, 3]'] + compose: + - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovun_{type[3]}", [b]]}, "{type[4]}"]] + + - name: "vqrdmulh{type[1]}" + doc: "Signed saturating rounding doubling multiply returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmulh]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i16", 'h_s16', 's16'] + - ["i32", 's_s32', 's32'] + compose: + - FnCall: [simd_extract!, [{FnCall: ["vqrdmulh_{type[2]}", [{FnCall: ["vdup_n_{type[2]}", [a]]}, {FnCall: ["vdup_n_{type[2]}", [b]]}]]}, '0']] + + - name: "vqrdmulh{type[2]}" + doc: "Signed saturating rounding doubling multiply returning high half" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmulh, LANE = 1]]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, 'h_lane_s16', 'h_s16', '2'] + - ["i16", int16x8_t, 'h_laneq_s16', 'h_s16', '3'] + - ["i32", int32x2_t, 's_lane_s32', 's_s32', '1'] + - ["i32", int32x4_t, 's_laneq_s32', 's_s32', '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] + - FnCall: ["vqrdmulh{type[3]}", [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] + + - name: "vqrdmlah{neon_type.no}" + doc: "Signed saturating rounding doubling multiply accumulate returning high half" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah]]}]] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + safety: + unsafe: [neon] + types: + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "vqrdmlah{neon_type.no}" + links: + - link: "llvm.aarch64.neon.sqrdmlah.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqrdmlah{type[3]}" + doc: "Signed saturating rounding doubling multiply accumulate returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah]]}]] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, s16, 'h_s16'] + - ["i32", int32x2_t, s32, 's_s32'] + compose: + - Let: [a, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [a]]}] + - Let: [b, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [b]]}] + - Let: [c, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [c]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrdmlah_{type[2]}", [a, b, c]]}, '0']] + + - name: "vqrdmlah{type[0]}" + doc: "Signed saturating rounding doubling multiply accumulate returning high half" + arguments: ["a: {type[1]}", "b: {type[2]}", "c: {neon_type[3]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_s16, int16x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_s16, int16x8_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s16, int16x8_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_s32, int32x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_s32, int32x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_s32, int32x4_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s32, int32x4_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - Let: [c, "{type[1]}", {FnCall: [simd_shuffle!, [c, c, "{type[5]}"]]}] + - FnCall: ["vqrdmlah{neon_type[2].no}", [a, b, c]] + + - name: "vqrdmlah{type[4]}" + doc: "Signed saturating rounding doubling multiply accumulate returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, '2', "h_s16", h_lane_s16, h_s16] + - ["i16", int16x8_t, '3', "h_s16", h_laneq_s16, h_s16] + - ["i32", int32x2_t, '1', "s_s32", s_lane_s32, s_s32] + - ["i32", int32x4_t, '2', "s_s32", s_laneq_s32, s_s32] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - FnCall: ["vqrdmlah{type[5]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vqrdmlsh{neon_type.no}" + doc: "Signed saturating rounding doubling multiply subtract returning high half" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh]]}]] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + safety: + unsafe: [neon] + types: + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "vqrdmlsh{neon_type.no}" + links: + - link: "llvm.aarch64.neon.sqrdmlsh.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqrdmlsh{type[1]}" + doc: "Signed saturating rounding doubling multiply subtract returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh]]}]] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + safety: + unsafe: [neon] + types: + - ["i16", "h_s16", int16x4_t, s16] + - ["i32", "s_s32", int32x2_t, s32] + compose: + - Let: [a, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [a]]}] + - Let: [b, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [b]]}] + - Let: [c, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [c]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrdmlsh_{type[3]}", [a, b, c]]}, '0']] + + - name: "vqrdmlsh{type[0]}" + doc: "Signed saturating rounding doubling multiply subtract returning high half" + arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}", "c: {neon_type[3]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh, LANE = 1]]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_s16, int16x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_s16, int16x8_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s16, int16x8_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_s32, int32x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_s32, int32x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_s32, int32x4_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s32, int32x4_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - Let: [c, "{type[1]}", {FnCall: [simd_shuffle!, [c, c, "{type[5]}"]]}] + - FnCall: ["vqrdmlsh{neon_type[2].no}", [a, b, c]] + + - name: "vqrdmlsh{type[3]}" + doc: "Signed saturating rounding doubling multiply subtract returning high half" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [target_feature, ['enable = "rdm"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh, LANE = 1]]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i16", int16x4_t, '2', h_lane_s16, h_s16] + - ["i16", int16x8_t, '3', h_laneq_s16, h_s16] + - ["i32", int32x2_t, '1', s_lane_s32, s_s32] + - ["i32", int32x4_t, '2', s_laneq_s32, s_s32] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - FnCall: ["vqrdmlsh{type[4]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vqrshl{type[0]}" + doc: "Signed saturating rounding shift left" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['s_s32', "i32"] + - ['d_s64', "i64"] + compose: + - LLVMLink: + name: "vqrshl{type[0]}" + links: + - link: "llvm.aarch64.neon.sqrshl.{type[1]}" + arch: aarch64,arm64ec + + - name: "vqrshl{type[1]}" + doc: "Signed saturating rounding shift left" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i8", 'b_s8', int8x8_t, s8] + - ["i16", 'h_s16', int16x4_t, s16] + compose: + - Let: [a, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [a]]}] + - Let: [b, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [b]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrshl_{type[3]}", [a, b]]}, '0']] + + - name: "vqrshl{type[2]}" + doc: "Unsigned signed saturating rounding shift left" + arguments: ["a: {type[0]}", "b: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u32", "i32", 's_u32'] + - ["u64", "i64", 'd_u64'] + compose: + - LLVMLink: + name: "vqrshl{type[2]}" + links: + - link: "llvm.aarch64.neon.uqrshl.{type[1]}" + arch: aarch64,arm64ec + + - name: "vqrshl{type[2]}" + doc: "Unsigned signed saturating rounding shift left" + arguments: ["a: {type[0]}", "b: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u8", "i8", "b_u8", uint8x8_t, int8x8_t, s8] + - ["u16", "i16", "h_u16", uint16x4_t, int16x4_t, s16] + compose: + - Let: [a, "{neon_type[3]}", {FnCall: ["vdup_n_{type[0]}", [a]]}] + - Let: [b, "{neon_type[4]}", {FnCall: ["vdup_n_{type[5]}", [b]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrshl_{type[0]}", [a, b]]}, '0']] + + - name: "vqrshrn{type[2]}" + doc: "Signed saturating rounded shift right narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i16", "i8", 'h_n_s16', 'N >= 1 && N <= 8', int16x8_t, q_n_s16] + - ["i32", "i16", 's_n_s32', 'N >= 1 && N <= 16', int32x4_t, q_n_s32] + - ["i64", "i32", 'd_n_s64', 'N >= 1 && N <= 32', int64x2_t, q_n_s64] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - Let: [a, "{neon_type[4]}", {FnCall: ["vdup{type[5]}", [a]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrshrn_n{neon_type[4].noq}::", [a]]}, '0']] + + - name: "vqrshrn{type[3]}" + doc: "Signed saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, '_high_n_s16', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]', 'N >= 1 && N <= 8'] + - [int16x4_t, int32x4_t, int16x8_t, '_high_n_s32', '[0, 1, 2, 3, 4, 5, 6, 7]', 'N >= 1 && N <= 16'] + - [int32x2_t, int64x2_t, int32x4_t, '_high_n_s64', '[0, 1, 2, 3]', 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[5]}"]] + - FnCall: [simd_shuffle!, [a, {FnCall: ["vqrshrn_n{neon_type[1].noq}::", [b]]}, "{type[4]}"]] + + - name: "vqrshrn{type[0]}" + doc: "Unsigned saturating rounded shift right narrow" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [h_n_u16, u16, u8, 'N >= 1 && N <= 8', uint16x8_t, q_n_u16, _n_u16] + - [s_n_u32, u32, u16, 'N >= 1 && N <= 16', uint32x4_t, q_n_u32, _n_u32] + - [d_n_u64, u64, u32, 'N >= 1 && N <= 32', uint64x2_t, q_n_u64, _n_u64] + compose: + - FnCall: [static_assert!, ['{type[3]}']] + - Let: [a, "{neon_type[4]}", {FnCall: ["vdup{type[5]}", [a]]}] + - FnCall: [simd_extract!, [{FnCall: ["vqrshrn{type[6]}::", [a]]}, '0']] + + - name: "vqrshrn_high_n{neon_type[1].noq}" + doc: "Unsigned saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ['{type[3]}']] + - FnCall: + - simd_shuffle! + - - a + - FnCall: + - "vqrshrn_n{neon_type[1].noq}::" + - - b + - "{type[4]}" + + - name: "vqrshrun{type[0]}" + doc: "Signed saturating rounded shift right unsigned narrow" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [h_n_s16, "i16", "u8", 'N >= 1 && N <= 8', int16x8_t, s16] + - [s_n_s32, "i32", "u16", 'N >= 1 && N <= 16', int32x4_t, s32] + - [d_n_s64, "i64", "u32", 'N >= 1 && N <= 32', int64x2_t, s64] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - Let: + - a + - "{neon_type[4]}" + - FnCall: ["vdupq_n_{type[5]}", [a]] + - FnCall: + - simd_extract! + - - FnCall: + - "vqrshrun_n_{type[5]}::" + - - a + - '0' + + - name: "vqrshrun_high_n{neon_type[1].noq}" + doc: "Signed saturating rounded shift right unsigned narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int16x8_t, uint8x16_t, 'N >= 1 && N <= 8', s16, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, int32x4_t, uint16x8_t, 'N >= 1 && N <= 16', s32, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, int64x2_t, uint32x4_t, 'N >= 1 && N <= 32', s64, '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: + - "vqrshrun_n_{type[4]}::" + - - b + - "{type[5]}" + + - name: "vqshld_{type}" + doc: "Signed saturating shift left" + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - i64 + compose: + - LLVMLink: + name: "vqshld{type}" + links: + - link: "llvm.aarch64.neon.sqshl.{type}" + arch: aarch64,arm64ec + + - name: "vqshl{type[0]}" + doc: "Signed saturating shift left" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [b_s8, "i8", int8x8_t] + - [h_s16, "i16", int16x4_t] + - [s_s32, "i32", int32x2_t] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: + - "vqshl{neon_type[2].noq}" + - - FnCall: ["vdup_n{neon_type[2].no}", [a]] + - FnCall: ["vdup_n{neon_type[2].no}", [b]] + - FnCall: [simd_extract!, [c, '0']] + + - name: "vqshl{type[0]}" + doc: "Signed saturating shift left" + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [b_n_s8, "i8", "3", s8] + - [h_n_s16, "i16", "4", s16] + - [s_n_s32, "i32", "5", s32] + - [d_n_s64, "i64", "6", s64] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] + - FnCall: + - simd_extract! + - - FnCall: + - "vqshl_n_{type[3]}::" + - - FnCall: ["vdup_n_{type[3]}", [a]] + - '0' + + - name: "vqshld_{type[0]}" + doc: "Unsigned saturating shift left" + arguments: ["a: {type[0]}", "b: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u64", "i64"] + compose: + - LLVMLink: + name: "vqshld{type[0]}" + links: + - link: "llvm.aarch64.neon.uqshl.{type[1]}" + arch: aarch64,arm64ec + + - name: "vqshl{type[0]}" + doc: "Unsigned saturating shift left" + arguments: ["a: {type[1]}", "b: {type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [b_u8, "u8", "i8", uint8x8_t, int8x8_t] + - [h_u16, "u16", "i16", uint16x4_t, int16x4_t] + - [s_u32, "u32", "i32", uint32x2_t, int32x2_t] + compose: + - Let: + - c + - "{neon_type[3]}" + - FnCall: + - "vqshl{neon_type[3].noq}" + - - FnCall: ["vdup{neon_type[3].N}", [a]] + - FnCall: ["vdup{neon_type[4].N}", [b]] + - FnCall: [simd_extract!, [c, '0']] + + - name: "vqshl{type[0]}" + doc: "Unsigned saturating shift left" + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [b_n_u8, "u8", '3'] + - [h_n_u16, "u16", '4'] + - [s_n_u32, "u32", '5'] + - [d_n_u64, "u64", '6'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] + - FnCall: + - simd_extract! + - - FnCall: ["vqshl_n_{type[1]}::", [{FnCall: ["vdup_n_{type[1]}", [a]]}]] + - '0' + + - name: "vqshrnd_n_s64" + doc: "Signed saturating shift right narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i64", "i32"] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vqshrnd{type[1]}" + arguments: + - "a: {type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.sqshrn.{type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqshrnd_n_s64", [a, N]] + + - name: "vqshrn{type[0]}" + doc: "Signed saturating shift right narrow" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [h_n_s16, "i16", "i8", 'N >= 1 && N <= 8', s16] + - [s_n_s32, "i32", "i16", 'N >= 1 && N <= 16', s32] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_extract! + - - FnCall: + - "vqshrn_n_{type[4]}::" + - - FnCall: ["vdupq_n_{type[4]}", [a]] + - '0' + + - name: "vqshrn{type[0]}" + doc: "Signed saturating shift right narrow" + arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] + return_type: "{neon_type[3]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_high_n_s16, int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]', s16] + - [_high_n_s32, int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]', s32] + - [_high_n_s64, int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]', s64] + compose: + - FnCall: [static_assert!, ["{type[4]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: ["vqshrn_n_{type[6]}::", [b]] + - "{type[5]}" + + - name: "vqshrnd_n_u64" + doc: "Unsigned saturating shift right narrow" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["u64", "u32"] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vqshrnd_n_u64" + arguments: + - "a: u64" + - "n: i32" + links: + - link: "llvm.aarch64.neon.uqshrn.i32" + arch: aarch64,arm64ec + - FnCall: ["_vqshrnd_n_u64", ["a.as_signed()", N]] + + - name: "vqshrn{type[0]}" + doc: "Unsigned saturating shift right narrow" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ['h_n_u16', "u16", "u8", 'N >= 1 && N <= 8'] + - ['s_n_u32', "u32", "u16", 'N >= 1 && N <= 16'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - "simd_extract!" + - - FnCall: + - "vqshrn_n_{type[1]}::" + - - FnCall: ["vdupq_n_{type[1]}", [a]] + - '0' + + - name: "vqshrn{type[0]}" + doc: "Unsigned saturating shift right narrow" + arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] + return_type: "{neon_type[3]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_high_n_u16, uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [_high_n_u32, uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [_high_n_u64, uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[4]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: ["vqshrn_n_{neon_type[2]}::", [b]] + - "{type[5]}" + + - name: "vqshrun{type[0]}" + doc: "Signed saturating shift right unsigned narrow" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [h_n_s16, "i16", "u8", 'N >= 1 && N <= 8', s16] + - [s_n_s32, "i32", "u16", 'N >= 1 && N <= 16', s32] + - [d_n_s64, "i64", "u32", 'N >= 1 && N <= 32', s64] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_extract! + - - FnCall: + - "vqshrun_n_{type[4]}::" + - - FnCall: ["vdupq_n_{type[4]}", [a]] + - '0' + + - name: "vqshrun_high_n_{neon_type[1]}" + doc: "Signed saturating shift right unsigned narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, int32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, int64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: ["vqshrun_n_{neon_type[1]}::", [b]] + - "{type[4]}" + + - name: "vsqadd{type[0]}" + doc: "Unsigned saturating accumulate of signed value" + arguments: ["a: {type[1]}", "b: {type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usqadd]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [b_u8, "u8", "i8", s8] + - [h_u16, "u16", "i16", s16] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vsqadd_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[2]}", [b]] + - '0' + + - name: "vsqadd{type[0]}" + doc: "Unsigned saturating accumulate of signed value" + arguments: ["a: {type[1]}", "b: {type[2]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usqadd]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [s_u32, "u32", "i32"] + - [d_u64, "u64", "i64"] + compose: + - LLVMLink: + name: "vsqadd{type[0]}" + links: + - link: "llvm.aarch64.neon.usqadd.{type[2]}" + arch: aarch64,arm64ec + + - name: "vsqrt{neon_type.no}" + doc: "Calculates the square root of each lane." + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fsqrt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_fsqrt, [a]] + + - name: "vrsqrts{type[0]}" + doc: "Floating-point reciprocal square root step" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrts]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [_f64, float64x1_t, v1f64] + - [q_f64, float64x2_t, v2f64] + compose: + - LLVMLink: + name: "vrsqrts{type[0]}" + links: + - link: "llvm.aarch64.neon.frsqrts.{type[2]}" + arch: aarch64,arm64ec + + - name: "vrsqrts{type[0]}" + doc: "Floating-point reciprocal square root step" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrts]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [s_f32, "f32"] + - [d_f64, "f64"] + compose: + - LLVMLink: + name: "vrsqrts{type[0]}" + links: + - link: "llvm.aarch64.neon.frsqrts.{type[1]}" + arch: aarch64,arm64ec + + - name: "vrecpe{type[0]}" + doc: "Reciprocal estimate." + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpe]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [_f64, float64x1_t, v1f64] + - [q_f64, float64x2_t, v2f64] + compose: + - LLVMLink: + name: "vrecpe{type[0]}" + links: + - link: "llvm.aarch64.neon.frecpe.{type[2]}" + arch: aarch64,arm64ec + + - name: "vrecpe{type[0]}" + doc: "Reciprocal estimate." + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpe]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [s_f32, "f32"] + - [d_f64, "f64"] + compose: + - LLVMLink: + name: "vrecpe{type[0]}" + links: + - link: "llvm.aarch64.neon.frecpe.{type[1]}" + arch: aarch64,arm64ec + + - name: "vrecps{type[0]}" + doc: "Floating-point reciprocal step" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecps]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [_f64, float64x1_t, v1f64] + - [q_f64, float64x2_t, v2f64] + compose: + - LLVMLink: + name: "vrecps{type[0]}" + links: + - link: "llvm.aarch64.neon.frecps.{type[2]}" + arch: aarch64,arm64ec + + - name: "vrecps{type[0]}" + doc: "Floating-point reciprocal step" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecps]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [s_f32, "f32"] + - [d_f64, "f64"] + compose: + - LLVMLink: + name: "vrecps{type[0]}" + links: + - link: "llvm.aarch64.neon.frecps.{type[1]}" + arch: aarch64,arm64ec + + - name: "vrecpx{type[0]}" + doc: "Floating-point reciprocal exponent" + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpx]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [s_f32, "f32"] + - [d_f64, "f64"] + compose: + - LLVMLink: + name: "vrecpxs{type[0]}" + links: + - link: "llvm.aarch64.neon.frecpx.{type[1]}" + arch: aarch64,arm64ec + + - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" + doc: Vector reinterpret cast operation + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: [*neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [poly64x1_t, int64x1_t] + - [poly64x1_t, uint64x1_t] + - [int64x1_t, poly64x1_t] + - [uint64x1_t, poly64x1_t] + - [poly64x2_t, int64x2_t] + - [poly64x2_t, uint64x2_t] + - [int64x2_t, poly64x2_t] + - [uint64x2_t, poly64x2_t] + - [float64x1_t, int8x8_t] + - [float64x1_t, int16x4_t] + - [float64x1_t, int32x2_t] + - [float64x1_t, int64x1_t] + - [float64x2_t, int8x16_t] + - [float64x2_t, int16x8_t] + - [float64x2_t, int32x4_t] + - [float64x2_t, int64x2_t] + - [float64x1_t, uint8x8_t] + - [float64x1_t, uint16x4_t] + - [float64x1_t, uint32x2_t] + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint8x16_t] + - [float64x2_t, uint16x8_t] + - [float64x2_t, uint32x4_t] + - [float64x2_t, uint64x2_t] + - [float64x1_t, poly8x8_t] + - [float64x1_t, poly16x4_t] + - [float32x2_t, poly64x1_t] + - [float64x1_t, poly64x1_t] + - [float64x2_t, poly8x16_t] + - [float64x2_t, poly16x8_t] + - [float32x4_t, poly64x2_t] + - [float64x2_t, poly64x2_t] + - [float64x2_t, p128] + - [int8x8_t, float64x1_t] + - [int16x4_t, float64x1_t] + - [int32x2_t, float64x1_t] + - [int64x1_t, float64x1_t] + - [int8x16_t, float64x2_t] + - [int16x8_t, float64x2_t] + - [int32x4_t, float64x2_t] + - [int64x2_t, float64x2_t] + - [poly8x8_t, float64x1_t] + - [uint16x4_t, float64x1_t] + - [uint32x2_t, float64x1_t] + - [uint64x1_t, float64x1_t] + - [poly8x16_t, float64x2_t] + - [uint16x8_t, float64x2_t] + - [uint32x4_t, float64x2_t] + - [uint64x2_t, float64x2_t] + - [uint8x8_t, float64x1_t] + - [poly16x4_t, float64x1_t] + - [poly64x1_t, float64x1_t] + - [poly64x1_t, float32x2_t] + - [uint8x16_t, float64x2_t] + - [poly16x8_t, float64x2_t] + - [poly64x2_t, float64x2_t] + - [poly64x2_t, float32x4_t] + - [p128, float64x2_t] + - [float32x2_t, float64x1_t] + - [float64x1_t, float32x2_t] + - [float32x4_t, float64x2_t] + - [float64x2_t, float32x4_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vrshld_s64" + doc: "Signed rounding shift left" + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - "i64" + compose: + - LLVMLink: + name: "vrshld_{type}" + links: + - link: "llvm.aarch64.neon.srshl.{type}" + arch: aarch64,arm64ec + + - name: "vrshld_{type[0]}" + doc: "Unsigned rounding shift left" + arguments: ["a: {type[0]}", "b: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u64", "i64"] + compose: + - LLVMLink: + name: "vrshld_{type[0]}" + links: + - link: "llvm.aarch64.neon.urshl.{type[1]}" + arch: aarch64,arm64ec + + - name: "vrshrd_n_s64" + doc: "Signed rounding shift right" + arguments: ["a: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - ["i64", 'N >= 1 && N <= 64', '-N as i64'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - FnCall: [vrshld_s64, [a, "{type[2]}"]] + + - name: "vrshrd_n_u64" + doc: "Unsigned rounding shift right" + arguments: ["a: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - "u64" + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - FnCall: ["vrshld_u64", [a, '-N as i64']] + + - name: "vrshrn_high_n_{neon_type[1]}" + doc: "Rounding shift right narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rshrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: ["vrshrn_n_{neon_type[1]}::", [b]] + - "{type[4]}" + + - name: "vrsubhn_high_{neon_type[1]}" + doc: "Rounding subtract returning high narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[3]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rsubhn2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] + - [uint8x8_t, uint16x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] + compose: + - Let: + - x + - "{neon_type[0]}" + - FnCall: ["vrsubhn_{neon_type[1]}", [b, c]] + - FnCall: [simd_shuffle!, [a, x, "{type[4]}"]] + + - name: "vcopy{neon_type[0].lane_nox}" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 0', 'LANE2 = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, int8x8_t, int8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int16x4_t, int16x4_t, int16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int32x2_t, int32x2_t, int32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint8x8_t, uint8x8_t, uint8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint16x4_t, uint16x4_t, uint16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint32x2_t, uint32x2_t, uint32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly8x8_t, poly8x8_t, poly8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly16x4_t, poly16x4_t, poly16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [float32x2_t, float32x2_t, float32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int8x16_t, int8x8_t, int8x16_t, '4', '3', ' let b: int8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int16x8_t, int16x4_t, int16x8_t, '3', '2', ' let b: int16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int32x4_t, int32x2_t, int32x4_t, '2', '1', ' let b: int32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint8x16_t, uint8x8_t, uint8x16_t, '4', '3', ' let b: uint8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint16x8_t, uint16x4_t, uint16x8_t, '3', '2', ' let b: uint16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint32x4_t, uint32x2_t, uint32x4_t, '2', '1', ' let b: uint32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly8x16_t, poly8x8_t, poly8x16_t, '4', '3', ' let b: poly8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly16x8_t, poly16x4_t, poly16x8_t, '3', '2', ' let b: poly16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE1, '{type[3]}']] + - FnCall: [static_assert_uimm_bits!, [LANE2, '{type[4]}']] + - Identifier: ["{type[5]}", Symbol] + + - name: "vcopy{neon_type[0].laneq_nox}" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 0', 'LANE2 = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: + unsafe: [neon] + types: + - [int8x16_t, int8x16_t, int8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int16x8_t, int16x8_t, int16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int32x4_t, int32x4_t, int32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int64x2_t, int64x2_t, int64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint8x16_t, uint8x16_t, uint8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint16x8_t, uint16x8_t, uint16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint32x4_t, uint32x4_t, uint32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint64x2_t, uint64x2_t, uint64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly8x16_t, poly8x16_t, poly8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly16x8_t, poly16x8_t, poly16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly64x2_t, poly64x2_t, poly64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [float32x4_t, float32x4_t, float32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [float64x2_t, float64x2_t, float64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int8x8_t, int8x16_t, int8x8_t, '3', '4', ' let a: int8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int16x4_t, int16x8_t, int16x4_t, '2', '3', ' let a: int16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [int32x2_t, int32x4_t, int32x2_t, '1', '2', ' let a: int32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint8x8_t, uint8x16_t, uint8x8_t, '3', '4', ' let a: uint8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint16x4_t, uint16x8_t, uint16x4_t, '2', '3', ' let a: uint16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint32x2_t, uint32x4_t, uint32x2_t, '1', '2', 'let a: uint32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly8x8_t, poly8x16_t, poly8x8_t, '3', '4', ' let a: poly8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly16x4_t, poly16x8_t, poly16x4_t, '2', '3', ' let a: poly16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [float32x2_t, float32x4_t, float32x2_t, '1', '2', ' let a: float32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE1, '{type[3]}']] + - FnCall: [static_assert_uimm_bits!, [LANE2, '{type[4]}']] + - Identifier: ["{type[5]}", Symbol] + + - name: "vcopyq_lane_{neon_type[0]}" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 1', 'LANE2 = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int64x1_t, 'let b: int64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [uint64x2_t, uint64x1_t, 'let b: uint64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [poly64x2_t, poly64x1_t, 'let b: poly64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + - [float64x2_t, float64x1_t, ' let b: float64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE1, '1']] + - FnCall: [static_assert!, ['LANE2 == 0']] + - Identifier: ['{type[2]}', Symbol] + + - name: "vcopyq_lane_f32" + doc: "Insert vector element from another vector element" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 1', 'LANE2 = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1', '3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE1: i32, const LANE2: i32'] + safety: + unsafe: [neon] + types: + - [float32x4_t, float32x2_t, ' let b: float32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE1, 2]] + - FnCall: [static_assert_uimm_bits!, [LANE2, 1]] + - Identifier: ["{type[2]}", Symbol] + + - name: "vcreate_f64" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u64", float64x1_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vset_lane_f64" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f64", float64x1_t, float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vsetq_lane_f64" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f64", float64x2_t, float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vshld_s64" + doc: "Signed Shift left" + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sshl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - "i64" + compose: + - FnCall: + - transmute + - - FnCall: + - vshl_s64 + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vshld_{type[0]}" + doc: "Unsigned Shift left" + arguments: ["a: {type[0]}", "b: {type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ushl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["u64", "i64"] + compose: + - FnCall: + - transmute + - - FnCall: + - vshl_u64 + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vshll_high_n_{neon_type[0]}" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sshll2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x16_t, int16x8_t, int8x8_t, 'N >= 0 && N <= 8', '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x8_t, int32x4_t, int16x4_t, 'N >= 0 && N <= 16', '[4, 5, 6, 7]'] + - [int32x4_t, int64x2_t, int32x2_t, 'N >= 0 && N <= 32', '[2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - FnCall: ["vshll_n_{neon_type[2]}::", [b]] + + - name: "vshll_high_n_{neon_type[0]}" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ushll2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x16_t, uint16x8_t, uint8x8_t, 'N >= 0 && N <= 8', '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x8_t, uint32x4_t, uint16x4_t, 'N >= 0 && N <= 16', '[4, 5, 6, 7]'] + - [uint32x4_t, uint64x2_t, uint32x2_t, 'N >= 0 && N <= 32', '[2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] + - FnCall: ["vshll_n_{neon_type[2]}::", [b]] + + - name: "vshrn_high_n_{neon_type[1]}" + doc: "Shift right narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [shrn2, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_shuffle! + - - a + - FnCall: ["vshrn_n_{neon_type[1]}::", [b]] + - "{type[4]}" + + - name: "vsm3partw1{neon_type.no}" + doc: "SM3PARTW1" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sm4"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3partw1]]}]] + - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] + safety: + unsafe: [neon] + types: + - uint32x4_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sm3partw1 + links: + - link: "llvm.aarch64.crypto.sm3partw1" + arch: aarch64,arm64ec + + - name: "vsm3partw2{neon_type.no}" + doc: "SM3PARTW2" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sm4"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3partw2]]}]] + - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] + safety: + unsafe: [neon] + types: + - uint32x4_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sm3partw2 + links: + - link: "llvm.aarch64.crypto.sm3partw2" + arch: aarch64,arm64ec + + - name: "vsm3ss1{neon_type.no}" + doc: "SM3SS1" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sm4"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3ss1]]}]] + - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] + safety: + unsafe: [neon] + types: + - uint32x4_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sm3ss1 + links: + - link: "llvm.aarch64.crypto.sm3ss1" + arch: aarch64,arm64ec + + - name: "vsm4ekey{neon_type.no}" + doc: "SM4 key" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sm4"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm4ekey]]}]] + - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] + safety: + unsafe: [neon] + types: + - uint32x4_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sm4ekey + links: + - link: "llvm.aarch64.crypto.sm4ekey" + arch: aarch64,arm64ec + + - name: "vsm4e{neon_type.no}" + doc: "SM4 encode" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sm4"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm4e]]}]] + - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] + safety: + unsafe: [neon] + types: + - uint32x4_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sm4e + links: + - link: "llvm.aarch64.crypto.sm4e" + arch: aarch64,arm64ec + + - name: "vrax1{neon_type.no}" + doc: "Rotate and exclusive OR" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rax1]]}]] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + safety: + unsafe: [neon] + types: + - uint64x2_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.rax1 + links: + - link: "llvm.aarch64.crypto.rax1" + arch: aarch64,arm64ec + + - name: "vsha512h{neon_type.no}" + doc: "SHA512 hash update part 1" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512h]]}]] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + safety: + unsafe: [neon] + types: + - uint64x2_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sha512h + links: + - link: "llvm.aarch64.crypto.sha512h" + arch: aarch64,arm64ec + + - name: "vsha512h2{neon_type.no}" + doc: "SHA512 hash update part 2" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512h2]]}]] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + safety: + unsafe: [neon] + types: + - uint64x2_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sha512h2 + links: + - link: "llvm.aarch64.crypto.sha512h2" + arch: aarch64,arm64ec + + - name: "vsha512su0{neon_type.no}" + doc: "SHA512 schedule update 0" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512su0]]}]] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + safety: + unsafe: [neon] + types: + - uint64x2_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sha512su0 + links: + - link: "llvm.aarch64.crypto.sha512su0" + arch: aarch64,arm64ec + + - name: "vsha512su1{neon_type.no}" + doc: "SHA512 schedule update 1" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,sha3"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512su1]]}]] + - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] + safety: + unsafe: [neon] + types: + - uint64x2_t + compose: + - LLVMLink: + name: llvm.aarch64.crypto.sha512su1 + links: + - link: "llvm.aarch64.crypto.sha512su1" + arch: aarch64,arm64ec + + - name: "vrnd32x{neon_type.no}" + doc: "Floating-point round to 32-bit integer, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32x]]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "vrnd32x{neon_type.no}" + links: + - link: "llvm.aarch64.neon.frint32x.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnd32x{neon_type.no}" + doc: "Floating-point round to 32-bit integer, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32x]]}]] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - LLVMLink: + name: "vrnd32x{neon_type.no}" + arguments: + - "a: f64" + return_type: "f64" + links: + - link: "llvm.aarch64.frint32x.f64" + arch: aarch64,arm64ec + - FnCall: + - transmute + - - FnCall: + - _vrnd32x_f64 + - - FnCall: [simd_extract!, [a, 0]] + + - name: "vrnd32z{neon_type.no}" + doc: "Floating-point round to 32-bit integer toward zero" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32z]]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "vrnd32z{neon_type.no}" + links: + - link: "llvm.aarch64.neon.frint32z.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnd32z{neon_type.no}" + doc: "Floating-point round to 32-bit integer toward zero" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32z]]}]] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - LLVMLink: + name: "vrnd32z{neon_type.no}" + arguments: + - "a: f64" + return_type: "f64" + links: + - link: "llvm.aarch64.frint32z.f64" + arch: aarch64,arm64ec + - FnCall: + - transmute + - - FnCall: [_vrnd32z_f64, [{FnCall: [simd_extract!, [a, 0]]}]] + + - name: "vrnd64x{neon_type.no}" + doc: "Floating-point round to 64-bit integer, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64x]]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "vrnd64x{neon_type.no}" + links: + - link: "llvm.aarch64.neon.frint64x.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnd64x{neon_type.no}" + doc: "Floating-point round to 64-bit integer, using current rounding mode" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64x]]}]] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - LLVMLink: + name: "vrnd64x{neon_type.no}" + arguments: + - "a: f64" + return_type: "f64" + links: + - link: "llvm.aarch64.frint64x.f64" + arch: aarch64,arm64ec + - FnCall: + - transmute + - - FnCall: [_vrnd64x_f64, [{FnCall: [simd_extract!, [a, 0]]}]] + + - name: "vrnd64z{neon_type.no}" + doc: "Floating-point round to 64-bit integer toward zero" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64z]]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + - float64x2_t + compose: + - LLVMLink: + name: "vrnd64z{neon_type.no}" + links: + - link: "llvm.aarch64.neon.frint64z.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrnd64z{neon_type.no}" + doc: "Floating-point round to 64-bit integer toward zero" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [target_feature, ['enable = "neon,frintts"']] + - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64z]]}]] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - LLVMLink: + name: "vrnd64z{neon_type.no}" + arguments: + - "a: f64" + return_type: "f64" + links: + - link: "llvm.aarch64.frint64z.f64" + arch: aarch64,arm64ec + - FnCall: + - transmute + - - FnCall: [_vrnd64z_f64, [{FnCall: [simd_extract!, [a, 0]]}]] + + - name: "vtrn1{neon_type[0].no}" + doc: Transpose vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [trn1]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [int8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] + - [int16x4_t, '[0, 4, 2, 6]'] + - [int16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [int32x4_t, '[0, 4, 2, 6]'] + - [uint8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [uint8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] + - [uint16x4_t, '[0, 4, 2, 6]'] + - [uint16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [uint32x4_t, '[0, 4, 2, 6]'] + - [poly8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [poly8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] + - [poly16x4_t, '[0, 4, 2, 6]'] + - [poly16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] + - [float32x4_t, '[0, 4, 2, 6]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vtrn1{neon_type[0].no}" + doc: Transpose vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, '[0, 2]'] + - [int64x2_t, '[0, 2]'] + - [uint32x2_t, '[0, 2]'] + - [uint64x2_t, '[0, 2]'] + - [poly64x2_t, '[0, 2]'] + - [float32x2_t, '[0, 2]'] + - [float64x2_t, '[0, 2]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vtrn2{neon_type[0].no}" + doc: Transpose vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [trn2]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [int8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [int16x4_t, '[1, 5, 3, 7]'] + - [int16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [int32x4_t, '[1, 5, 3, 7]'] + - [uint8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [uint8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [uint16x4_t, '[1, 5, 3, 7]'] + - [uint16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [uint32x4_t, '[1, 5, 3, 7]'] + - [poly8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [poly8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [poly16x4_t, '[1, 5, 3, 7]'] + - [poly16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [float32x4_t, '[1, 5, 3, 7]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vtrn2{neon_type[0].no}" + doc: Transpose vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, '[1, 3]'] + - [int64x2_t, '[1, 3]'] + - [uint32x2_t, '[1, 3]'] + - [uint64x2_t, '[1, 3]'] + - [poly64x2_t, '[1, 3]'] + - [float32x2_t, '[1, 3]'] + - [float64x2_t, '[1, 3]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vzip2{neon_type[0].no}" + doc: Zip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [int8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [int16x4_t, '[2, 6, 3, 7]'] + - [int16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [int32x2_t, '[1, 3]'] + - [int32x4_t, '[2, 6, 3, 7]'] + - [int64x2_t, '[1, 3]'] + - [uint8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [uint8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [uint16x4_t, '[2, 6, 3, 7]'] + - [uint16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [uint32x2_t, '[1, 3]'] + - [uint32x4_t, '[2, 6, 3, 7]'] + - [uint64x2_t, '[1, 3]'] + - [poly8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [poly8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [poly16x4_t, '[2, 6, 3, 7]'] + - [poly16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [poly64x2_t, '[1, 3]'] + - [float32x2_t, '[1, 3]'] + - [float32x4_t, '[2, 6, 3, 7]'] + - [float64x2_t, '[1, 3]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vzip1{neon_type[0].no}" + doc: Zip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [int8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] + - [int16x4_t, '[0, 4, 1, 5]'] + - [int16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [int32x2_t, '[0, 2]'] + - [int32x4_t, '[0, 4, 1, 5]'] + - [int64x2_t, '[0, 2]'] + - [uint8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [uint8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] + - [uint16x4_t, '[0, 4, 1, 5]'] + - [uint16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [uint32x2_t, '[0, 2]'] + - [uint32x4_t, '[0, 4, 1, 5]'] + - [uint64x2_t, '[0, 2]'] + - [poly8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [poly8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] + - [poly16x4_t, '[0, 4, 1, 5]'] + - [poly16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] + - [poly64x2_t, '[0, 2]'] + - [float32x2_t, '[0, 2]'] + - [float32x4_t, '[0, 4, 1, 5]'] + - [float64x2_t, '[0, 2]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vuzp1{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, '[0, 2]'] + - [int64x2_t, '[0, 2]'] + - [uint32x2_t, '[0, 2]'] + - [uint64x2_t, '[0, 2]'] + - [poly64x2_t, '[0, 2]'] + - [float32x2_t, '[0, 2]'] + - [float64x2_t, '[0, 2]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vuzp1{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uzp1]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [int8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] + - [int16x4_t, '[0, 2, 4, 6]'] + - [int16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [int32x4_t, '[0, 2, 4, 6]'] + - [uint8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [uint8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] + - [uint16x4_t, '[0, 2, 4, 6]'] + - [uint16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [uint32x4_t, '[0, 2, 4, 6] '] + - [poly8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [poly8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] + - [poly16x4_t, '[0, 2, 4, 6]'] + - [poly16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] + - [float32x4_t, '[0, 2, 4, 6]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vuzp2{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, '[1, 3]'] + - [int64x2_t, '[1, 3]'] + - [uint32x2_t, '[1, 3]'] + - [uint64x2_t, '[1, 3]'] + - [poly64x2_t, '[1, 3]'] + - [float32x2_t, '[1, 3]'] + - [float64x2_t, '[1, 3]'] + compose: + - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] + + - name: "vuzp2{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uzp2]]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [int8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [int16x4_t, '[1, 3, 5, 7]'] + - [int16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [int32x4_t, '[1, 3, 5, 7]'] + - [uint8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [uint8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [uint16x4_t, '[1, 3, 5, 7]'] + - [uint16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [uint32x4_t, '[1, 3, 5, 7]'] + - [poly8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [poly8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [poly16x4_t, '[1, 3, 5, 7]'] + - [poly16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [float32x4_t, '[1, 3, 5, 7]'] + compose: + - FnCall: + - "simd_shuffle!" + - - a + - b + - "{type[1]}" + + - name: "vabal_high_{neon_type[1]}" + doc: "Unsigned Absolute difference and Accumulate Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uabal]]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]', '[4, 5, 6, 7]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]', '[2, 3]'] + compose: + - Let: + - d + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] + - Let: + - e + - "{neon_type[2]}" + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + - Let: [f, "{neon_type[2]}", {FnCall: ["vabd_{neon_type[2]}", [d, e]]}] + - FnCall: + - simd_add + - - a + - FnCall: [simd_cast, [f]] + + - name: "vabal_high{neon_type[1].noq}" + doc: Signed Absolute difference and Accumulate Long + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sabal]]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x16_t, int8x16_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int8x8_t, uint8x8_t] + - [int32x4_t, int16x8_t, int16x8_t, '[4, 5, 6, 7]', int16x4_t, uint16x4_t] + - [int64x2_t, int32x4_t, int32x4_t, '[2, 3]', int32x2_t, uint32x2_t] + compose: + - Let: + - d + - "{neon_type[4]}" + - FnCall: + - simd_shuffle! + - - b + - b + - "{type[3]}" + - Let: + - e + - "{neon_type[4]}" + - FnCall: + - simd_shuffle! + - - c + - c + - "{type[3]}" + - Let: + - f + - "{neon_type[4]}" + - FnCall: + - "vabd{neon_type[4].no}" + - - d + - e + - Let: + - f + - "{neon_type[5]}" + - FnCall: + - simd_cast + - - f + - FnCall: + - simd_add + - - a + - FnCall: + - simd_cast + - - f + + - name: "vqabs{neon_type.no}" + doc: Signed saturating Absolute value + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-stable + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] + safety: + unsafe: [neon] + types: + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "sqabs.{neon_type}" + links: + - link: "llvm.aarch64.neon.sqabs.{neon_type}" + arch: aarch64,arm64ec + + - name: "vslid_n_{type}" + doc: Shift left and insert + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + static_defs: + - "const N: i32" + attr: + - *neon-stable + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sli, 'N = 2']]}]] + safety: + unsafe: [neon] + types: + - i64 + - u64 + compose: + - FnCall: + - "static_assert!" + - - 'N >= 0 && N <= 63' + - FnCall: + - transmute + - - FnCall: + - "vsli_n_{type}::" + - - FnCall: + - transmute + - - a + - FnCall: + - transmute + - - b + + - name: "vsrid_n_{type}" + doc: Shift right and insert + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + static_defs: + - "const N: i32" + attr: + - *neon-stable + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sri, 'N = 2']]}]] + safety: + unsafe: [neon] + types: + - i64 + - u64 + compose: + - FnCall: + - "static_assert!" + - - 'N >= 1 && N <= 64' + - FnCall: + - transmute + - - FnCall: + - "vsri_n_{type}::" + - - FnCall: + - transmute + - - a + - FnCall: + - transmute + - - b + + - name: "vpmaxnm{neon_type.no}" + doc: "Floating-point Maximum Number Pairwise (vector)." + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmaxnmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float32x2_t + - float64x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vpmaxnm{neon_type}" + links: + - link: "llvm.aarch64.neon.fmaxnmp.{neon_type}" + arch: aarch64,arm64ec + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['*mut f64', float64x1x2_t, float64x1_t] + - ['*mut f64', float64x2x2_t, float64x2_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'a']] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['*mut f64', float64x1x3_t, float64x1_t] + - ['*mut f64', float64x2x3_t, float64x2_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "c: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'a']] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['*mut f64', float64x1x4_t, float64x1_t] + - ['*mut f64', float64x2x4_t, float64x2_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "c: {neon_type[2]}" + - "d: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'b.3', 'a']] + + - name: "vfma{type[3]}" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '1', '_lane_f32'] + - [float32x2_t, float32x4_t, '2', '_laneq_f32'] + - [float32x4_t, float32x2_t, '1', 'q_lane_f32'] + - [float32x4_t, float32x4_t, '2', 'q_laneq_f32'] + - [float64x2_t, float64x2_t, '1', 'q_laneq_f64'] + compose: + - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] + - FnCall: + - "vfma{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vfma_lane_f64" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - FnCall: ["static_assert!", ["LANE == 0"]] + - FnCall: + - "vfma{neon_type.no}" + - - a + - b + - FnCall: ["vdup{neon_type.N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vfma_laneq_f64" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x1_t, float64x2_t] + compose: + - FnCall: ["static_assert_uimm_bits!", ["LANE", "1"]] + - FnCall: + - "vfma{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vfmaq_lane_f64" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmla, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x2_t, float64x1_t] + compose: + - FnCall: ["static_assert!", ["LANE == 0"]] + - FnCall: + - "vfma{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] + + - name: "vfma{type[2]}" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f32", float32x2_t, "s_lane_f32", '1'] + - ["f32", float32x4_t, "s_laneq_f32", '2'] + - ["f64", float64x2_t, "d_laneq_f64", '1'] + compose: + - LLVMLink: + name: "_vfma{type[2]}" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] + links: + - link: "llvm.fma.{type[0]}" + arch: aarch64,arm64ec + - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] + - Let: [c, "{type[0]}", {FnCall: [simd_extract!, [c, 'LANE as u32']]}] + - FnCall: ["_vfma{type[2]}", [b, c, a]] + + - name: "vfmad_lane_f64" + doc: "Floating-point fused multiply-add to accumulator" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f64", float64x1_t] + compose: + - LLVMLink: + name: "_vfmad_lane_f64" + arguments: ["a: f64", "b: f64", "c: f64"] + links: + - link: "llvm.fma.{type[0]}" + arch: aarch64,arm64ec + - FnCall: [static_assert!, ['LANE == 0']] + - Let: [c, "{type[0]}", {FnCall: [simd_extract!, [c, 'LANE as u32']]}] + - FnCall: ["_vfmad_lane_f64", [b, c, a]] + + - name: "vfms_f64" + doc: "Floating-point fused multiply-subtract from accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] + - FnCall: [vfma_f64, [a, b, c]] + + - name: "vfms{neon_type.no}" + doc: "Floating-point fused multiply-subtract from accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x2_t + compose: + - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] + - FnCall: [vfmaq_f64, [a, b, c]] + + - name: "vmls{neon_type.no}" + doc: "Floating-point multiply-subtract from accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - FnCall: [simd_sub, [a, {FnCall: [simd_mul, [b, c]]}]] + + - name: "vfms{type[3]}" + doc: "Floating-point fused multiply-subtract to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '1', _lane_f32] + - [float32x2_t, float32x4_t, '2', _laneq_f32] + - [float32x4_t, float32x2_t, '1', q_lane_f32] + - [float32x4_t, float32x4_t, '2', q_laneq_f32] + - [float64x2_t, float64x2_t, '1', q_laneq_f64] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[2]}']] + - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] + + - name: "vfms_lane_f64" + doc: "Floating-point fused multiply-subtract to accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: ["vfms{neon_type.no}", [a, b, {FnCall: ["vdup{neon_type.N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] + + - name: "vfms_laneq_f64" + doc: "Floating-point fused multiply-subtract to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x1_t, float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] + + - name: "vfmsq_lane_f64" + doc: "Floating-point fused multiply-subtract to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x2_t, float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] + + - name: "vfms{type[2]}" + doc: "Floating-point fused multiply-subtract to accumulator" + arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f32", float32x2_t, "s_lane_f32"] + - ["f32", float32x4_t, "s_laneq_f32"] + - ["f64", float64x1_t, "d_lane_f64"] + - ["f64", float64x2_t, "d_laneq_f64"] + compose: + - FnCall: ["vfma{type[2]}::", ['a', '-b', 'c']] + + - name: "vceqz{neon_type[0].no}" + doc: "Floating-point compare bitwise equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmeq]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, 'f32x2', 'f32x2::new(0.0, 0.0)'] + - [float32x4_t, uint32x4_t, 'f32x4', 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] + - [float64x1_t, uint64x1_t, 'f64', '0.0'] + - [float64x2_t, uint64x2_t, 'f64x2', 'f64x2::new(0.0, 0.0)'] + compose: + - Let: [b, '{type[2]}', '{type[3]}'] + - FnCall: [simd_eq, [a, {FnCall: [transmute, [b]]}]] + + - name: "vceqz{type[2]}" + doc: "Floating-point compare bitwise equal to zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", "s_f32"] + - ["f64", "u64", "d_f64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vceqz_{type[0]}" + - - FnCall: ["vdup_n_{type[0]}", [a]] + - '0' + + - name: "vceqzd_{type[2]}" + doc: "Compare bitwise equal to zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64", "s64"] + - ["u64", "u64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - "vceqz_{type[2]}" + - - FnCall: [transmute, [a]] + + - name: "vceqz{neon_type[0].no}" + doc: "Signed compare bitwise equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + - [poly8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [poly8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [poly64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [poly64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_eq + - - a + - FnCall: [transmute, [b]] + + - name: "vceqz{neon_type[0].no}" + doc: "Unsigned compare bitwise equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint8x8_t, u8x8, 'u8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint8x16_t, uint8x16_t, u8x16, 'u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint16x4_t, uint16x4_t, u16x4, 'u16x4::new(0, 0, 0, 0)'] + - [uint16x8_t, uint16x8_t, u16x8, 'u16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint32x2_t, uint32x2_t, u32x2, 'u32x2::new(0, 0)'] + - [uint32x4_t, uint32x4_t, u32x4, 'u32x4::new(0, 0, 0, 0)'] + - [uint64x1_t, uint64x1_t, u64x1, 'u64x1::new(0)'] + - [uint64x2_t, uint64x2_t, u64x2, 'u64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_eq + - - a + - FnCall: [transmute, [b]] + + - name: "vcge{neon_type.no}" + doc: "Compare unsigned greater than or equal" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhs]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - uint64x1_t + - uint64x2_t + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vcge{type[0]}" + doc: "Floating-point compare greater than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vcge_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - FnCall: ["vdup_n_{type[1]}", [b]] + - '0' + + - name: "vcge{neon_type[0].no}" + doc: "Floating-point compare greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vcge{type[0]}" + doc: "Compare greater than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["d_s64", "i64", "u64", s64] + - ["d_u64", "u64", "u64", u64] + compose: + - FnCall: + - transmute + - - FnCall: + - "vcge_{type[3]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vclt{neon_type.no}" + doc: "Compare unsigned less than" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhi]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - uint64x1_t + - uint64x2_t + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vcltd_{type[0]}" + doc: "Compare less than" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s64", "i64", "u64"] + - ["u64", "u64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - "vclt_{type[0]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vtst{neon_type[0].no}" + doc: "Unsigned compare bitwise Test bits nonzero" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmtst]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint64x1_t, u64x1, 'u64x1::new(0)'] + - [uint64x2_t, u64x2, 'u64x2::new(0, 0)'] + compose: + - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] + - Let: [d, "{type[1]}", "{type[2]}"] + - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] + + - name: "vcgez{neon_type[0].no}" + doc: "Floating-point compare greater than or equal to zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] + - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] + - [float64x1_t, uint64x1_t, f64, '0.0'] + - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_ge + - - a + - FnCall: [transmute, [b]] + + - name: "vcgez{type[0]}" + doc: "Floating-point compare greater than or equal to zero" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vcgez_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - '0' + + - name: "vclezd_s64" + doc: "Compare less than or equal to zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: [vclez_s64, [{FnCall: [transmute, [a]]}]] + + - name: "vcgtd_{type[2]}" + doc: "Compare greater than" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64", 's64'] + - ["u64", "u64", 'u64'] + compose: + - FnCall: + - transmute + - - FnCall: + - "vcgt_{type[2]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vcgtz{neon_type[0].no}" + doc: "Compare signed greater than zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] + - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_gt + - - a + - FnCall: [transmute, [b]] + + - name: "vcgtzd_s64" + doc: "Compare signed greater than zero" + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["i64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - vcgtz_s64 + - - FnCall: [transmute, [a]] + + - name: "vcgtz{neon_type[0].no}" + doc: "Floating-point compare greater than zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] + - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] + - [float64x1_t, uint64x1_t, f64, '0.0'] + - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] + compose: + - Let: [b, "{type[2]}", "{type[3]}"] + - FnCall: [simd_gt, [a, {FnCall: [transmute, [b]]}]] + + - name: "vcgtz{type[0]}" + doc: "Floating-point compare greater than zero" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32", "u32"] + - ["d_f64", "f64", "u64"] + compose: + - FnCall: + - "simd_extract!" + - - FnCall: + - "vcgtz_{type[1]}" + - - FnCall: ["vdup_n_{type[1]}", [a]] + - '0' + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcvt{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vmul{neon_type[0].N}" + doc: "Vector multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, "f64"] + - [float64x2_t, "f64"] + compose: + - FnCall: + - simd_mul + - - a + - FnCall: ["vdup{neon_type[0].N}", [b]] + + - name: "vmul_lane_f64" + doc: "Floating-point multiply" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - float64x1_t + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - simd_mul + - - a + - FnCall: + - "transmute::" + - - FnCall: [simd_extract!, [b, 'LANE as u32']] + + - name: "vmulq_lane_f64" + doc: "Floating-point multiply" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x2_t, float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: + - simd_mul + - - a + - FnCall: ["simd_shuffle!", [b, b, '[LANE as u32, LANE as u32]']] + + - name: "vmuld_lane_f64" + doc: "Floating-point multiply" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f64", float64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Let: [b, '{type[0]}', {FnCall: [simd_extract!, [b, 'LANE as u32']]}] + - Identifier: ['a * b', Symbol] + + - name: "vmul_laneq_f64" + doc: "Floating-point multiply" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x1_t, float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: + - simd_mul + - - a + - FnCall: + - "transmute::" + - - FnCall: [simd_extract!, [b, 'LANE as u32']] + + - name: "vmulq_laneq_f64" + doc: "Floating-point multiply" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float64x2_t, float64x2_t, float64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: + - simd_mul + - - a + - FnCall: [simd_shuffle!, [b, b, '[LANE as u32, LANE as u32]']] + + - name: "vmul{type[2]}" + doc: "Floating-point multiply" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["f32", float32x2_t, "s_lane_f32", '1'] + - ["f32", float32x4_t, "s_laneq_f32", '2'] + - ["f64", float64x2_t, "d_laneq_f64", '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - Let: [b, '{type[0]}', {FnCall: [simd_extract!, [b, 'LANE as u32']]}] + - Identifier: ['a * b', Symbol] + + - name: "vrsrad_n_s64" + doc: "Signed rounding shift right and accumulate." + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - "i64" + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - Let: [b, "{type}", {FnCall: ["vrshrd_n_s64::", [b]]}] + - Identifier: ['a.wrapping_add(b)', Symbol] + + - name: "vmlsl_high_n_{neon_type[1]}" + doc: "Multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, "i16"] + - [int64x2_t, int32x4_t, "i32"] + compose: + - FnCall: ["vmlsl_high_{neon_type[1]}", [a, b, {FnCall: ["vdupq_n_{neon_type[1]}", [c]]}]] + + - name: "vmlsl_high_n_{neon_type[1]}" + doc: "Multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x8_t, "u16"] + - [uint64x2_t, uint32x4_t, "u32"] + compose: + - FnCall: ["vmlsl_high_{neon_type[1]}", [a, b, {FnCall: ["vdupq_n_{neon_type[1]}", [c]]}]] + + - name: "vmlsl_high_lane{neon_type[2].no}" + doc: "Multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int64x2_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int64x2_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmlsl_high_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmlsl_high_lane{neon_type[2].no}" + doc: "Multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmlsl_high_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vclt{neon_type[0].no}" + doc: "Floating-point compare less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vclt{type[2]}" + doc: "Floating-point compare less than" + arguments: ["a: {type[0]}", "b: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["f32", "u32", 's_f32'] + - ["f64", "u64", 'd_f64'] + compose: + - FnCall: + - simd_extract! + - - FnCall: + - "vclt_{type[0]}" + - - FnCall: ["vdup_n_{type[0]}", [a]] + - FnCall: ["vdup_n_{type[0]}", [b]] + - '0' + + - name: "vabdl_high_{neon_type[0]}" + doc: "Unsigned Absolute difference Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uabdl]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint8x16_t, uint16x8_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x8_t, uint32x4_t, uint16x4_t, '[4, 5, 6, 7]'] + - [uint32x4_t, uint64x2_t, uint32x2_t, '[2, 3]'] + compose: + - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[3]}"]]}] + - Let: [d, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - FnCall: [simd_cast, [{FnCall: ["vabd_{neon_type[0]}", [c, d]]}]] + + - name: "vfms_n_f64" + doc: "Floating-point fused Multiply-subtract to accumulator(vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, "f64"] + compose: + - FnCall: + - "vfms{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [c]] + + - name: "vfmsq_n_f64" + doc: "Floating-point fused Multiply-subtract to accumulator(vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x2_t, "f64"] + compose: + - FnCall: + - "vfms{neon_type[1].no}" + - - a + - b + - FnCall: ["vdup{neon_type[1].N}", [c]] + + - name: "vpminnm{type[0]}" + doc: "Floating-point minimum number pairwise" + arguments: ["a: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['s_f32', float32x2_t, "f32"] + - ['qd_f64', float64x2_t, "f64"] + compose: + - LLVMLink: + name: "vpminnm{type[0]}" + links: + - link: "llvm.aarch64.neon.fminnmv.{type[2]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vpmaxnm{type[0]}" + doc: "Floating-point maximum number pairwise" + arguments: ["a: {neon_type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmaxnmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ['s_f32', float32x2_t, "f32"] + - ['qd_f64', float64x2_t, "f64"] + compose: + - LLVMLink: + name: "vpmaxnm{type[0]}" + links: + - link: "llvm.aarch64.neon.fmaxnmv.{type[2]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vcled_{type[0]}" + doc: "Compare less than or equal" + arguments: ["a: {type[1]}", "b: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s64", "i64", "u64"] + - ["u64", "u64", "u64"] + compose: + - FnCall: + - transmute + - - FnCall: + - "vcle_{type[0]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vqdmulh{neon_type[0].lane_nox}" + doc: "Vector saturating doubling multiply high by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, '2'] + - [int16x8_t, int16x4_t, '2'] + - [int32x2_t, int32x2_t, '1'] + - [int32x4_t, int32x2_t, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - FnCall: + - "vqdmulh{neon_type[0].no}" + - - a + - FnCall: + - "vdup{neon_type[0].N}" + - - FnCall: [simd_extract!, [b, 'LANE as u32']] + + - name: "vqabs{type[2]}" + doc: "Signed saturating absolute value" + arguments: ["a: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] + safety: + unsafe: [neon] + types: + - ["i8", "s8", 'b_s8'] + - ["i16", "s16", 'h_s16'] + compose: + - FnCall: + - "simd_extract!" + - - FnCall: ["vqabs_{type[1]}", [{FnCall: ["vdup_n_{type[1]}", [a]]}]] + - '0' + + - name: "vqabs{type[1]}" + doc: "Signed saturating absolute value" + arguments: ["a: {type[0]}"] + return_type: "{type[0]}" + attr: + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] + safety: + unsafe: [neon] + types: + - ["i32", "s_s32"] + - ["i64", "d_s64"] + compose: + - LLVMLink: + name: "vqabs{type[1]}" + links: + - link: "llvm.aarch64.neon.sqabs.{type[0]}" + arch: aarch64,arm64ec + + - name: "vmull_high_n_{neon_type[0]}" + doc: "Multiply long" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smull2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int16x8_t, "i16", int32x4_t] + - [int32x4_t, "i32", int64x2_t] + compose: + - FnCall: + - "vmull_high_{neon_type[0]}" + - - a + - FnCall: ["vdupq_n_{neon_type[0]}", [b]] + + - name: "vmull_high_n_{neon_type[0]}" + doc: "Multiply long" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umull2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint16x8_t, "u16", uint32x4_t] + - [uint32x4_t, "u32", uint64x2_t] + compose: + - FnCall: + - "vmull_high_{neon_type[0]}" + - - a + - FnCall: ["vdupq_n_{neon_type[0]}", [b]] + + - name: "vmull_high_lane{neon_type[1].no}" + doc: "Multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smull2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int16x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int16x8_t, int16x8_t, int32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int32x2_t, int64x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int32x4_t, int64x2_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmull_high_{neon_type[0]}" + - - a + - FnCall: [simd_shuffle!, [b, b, '{type[4]}']] + + - name: "vmull_high_lane{neon_type[1].no}" + doc: "Multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umull2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x8_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmull_high_{neon_type[0]}" + - - a + - FnCall: [simd_shuffle!, [b, b, '{type[4]}']] + + - name: "vrsqrte{neon_type.no}" + doc: "Reciprocal square-root estimate." + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrte]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float64x1_t + - float64x2_t + compose: + - LLVMLink: + name: "vrsqrte{neon_type.no}" + links: + - link: "llvm.aarch64.neon.frsqrte.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrsqrte{type[0]}" + doc: "Reciprocal square-root estimate." + arguments: ["a: {type[1]}"] + return_type: "{type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrte]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["s_f32", "f32"] + - ["d_f64", "f64"] + compose: + - LLVMLink: + name: "vrsqrte{neon_type[1].no}" + links: + - link: "llvm.aarch64.neon.frsqrte.{type[1]}" + arch: aarch64,arm64ec + + - name: "vpminnm{neon_type.no}" + doc: "Floating-point Minimum Number Pairwise (vector)." + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - float32x2_t + - float64x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vpminnm{neon_type.no}" + links: + - link: "llvm.aarch64.neon.fminnmp.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqshlu{type[0]}" + doc: "Signed saturating shift left unsigned" + arguments: ["a: {type[1]}"] + return_type: "{type[2]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshlu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [b_n_s8, i8, u8, '3', s8] + - [h_n_s16, i16, u16, '4', s16] + - [s_n_s32, i32, u32, '5', s32] + - [d_n_s64, i64, u64, '6', s64] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: + - simd_extract! + - - FnCall: + - "vqshlu_n_{type[4]}::" + - - FnCall: ["vdup_n_{type[4]}", [a]] + - '0' + + - name: "vcvta{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to away" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtau]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + - [float64x1_t, uint64x1_t] + - [float64x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "vcvta{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.fcvtau.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to signed fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [float64x1_t, int64x1_t] + - [float64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vcvt{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vmlal_high_n_{neon_type[1]}" + doc: "Multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, "i16"] + - [int64x2_t, int32x4_t, "i32"] + compose: + - FnCall: + - "vmlal_high_{neon_type[1]}" + - - a + - b + - FnCall: ["vdupq_n_{neon_type[1]}", [c]] + + - name: "vmlal_high_n_{neon_type[1]}" + doc: "Multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x8_t, "u16"] + - [uint64x2_t, uint32x4_t, "u32"] + compose: + - FnCall: + - "vmlal_high_{neon_type[1]}" + - - a + - b + - FnCall: ["vdupq_n_{neon_type[1]}", [c]] + + - name: "vmlal_high_lane{neon_type[2].no}" + doc: "Multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int64x2_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int64x2_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: ['vmlal_high_{neon_type[2]}', [a, b, {FnCall: [simd_shuffle!, [c, c, '{type[4]}']]}]] + + - name: "vmlal_high_lane{neon_type[2].no}" + doc: "Multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: ['vmlal_high_{neon_type[2]}', [a, b, {FnCall: [simd_shuffle!, [c, c, '{type[4]}']]}]] + + - name: "vrsrad_n_u64" + doc: "Unsigned rounding shift right and accumulate." + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - "u64" + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 64']] + - Let: [b, u64, {FnCall: ["vrshrd_n_u64::", [b]]}] + - Identifier: ['a.wrapping_add(b)', Symbol] + + - name: "vcle{neon_type.no}" + doc: "Compare unsigned less than or equal" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhs]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - uint64x1_t + - uint64x2_t + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x2x4_t, "v2i64", "p0i64"] + - ["*const f64", float64x1x4_t, "v1f64", "p0f64"] + - ["*const f64", float64x2x4_t, "v2f64", "p0f64"] + compose: + - LLVMLink: + name: "vld4{neon_type[1].dup_nox}" + arguments: + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.ld4r.{type[2]}.{type[3]}" + arch: aarch64,arm64ec + - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as _']] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x2x4_t, "q_dup_s64"] + compose: + - FnCall: + - transmute + - - FnCall: ["vld4{type[2]}", [{FnCall: [transmute, [a]]}]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x2x4_t, "q_dup_s64"] + compose: + - FnCall: + - transmute + - - FnCall: ["vld4{type[2]}", [{FnCall: [transmute, [a]]}]] + diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml new file mode 100644 index 00000000000..a3c26a709b8 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml @@ -0,0 +1,9789 @@ +arch_cfgs: + - arch_name: aarch64 + target_feature: [neon] + llvm_prefix: llvm.aarch64.neon + +# Repeatedly used anchors +# #[stable(feature = "neon_intrinsics", since = "1.59.0")] +neon-stable: &neon-stable + FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + +# #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] +neon-unstable: &neon-unstable + FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + +# #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] +neon-v7: &neon-v7 + FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v7"']]} ]] + +# #[target_feature(enable = "neon,v7")] +enable-v7: &enable-v7 + FnCall: [target_feature, ['enable = "neon,v7"']] + +# #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] +neon-v8: &neon-v8 + FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v8"']]} ]] + +target-is-arm: &target-is-arm + FnCall: [cfg, ['target_arch = "arm"']] + +# #[cfg(not(target_arch = "arm"))] +target-not-arm: &target-not-arm + FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm"']]}]] + +neon-target-aarch64-arm64ec: &neon-target-aarch64-arm64ec + FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]] + +# #[cfg_attr(not(target_arch = "arm"), stable(feature = "neon_intrinsics", since = "1.59.0"))] +neon-stable-not-arm: &neon-stable-not-arm + FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']]}, *neon-stable]] + +#[cfg_attr(target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800"))] +neon-unstable-is-arm: &neon-unstable-is-arm + FnCall: [ cfg_attr, ['target_arch = "arm"', *neon-unstable]] + +# #[cfg_attr(all(test, not(target_env = "msvc"))] +msvc-disabled: &msvc-disabled + FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]] + +# all(test, target_arch = "arm") +test-is-arm: &test-is-arm + FnCall: [all, [test, 'target_arch = "arm"']] + +# #[target_feature(enable = "neon,aes")] +neon-aes: &neon-aes + FnCall: [target_feature, ['enable = "neon,aes"']] + +# #[target_feature(enable = "neon,i8mm")] +neon-i8mm: &neon-i8mm + FnCall: [target_feature, ['enable = "neon,i8mm"']] + +#[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_i8mm", issue = "117223"))] +neon-unstable-i8mm: &neon-unstable-i8mm + FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']] }, { FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] } ]] + +# #[unstable(feature = "stdarch_neon_fcma", issue = "117222")] +neon-unstable-fcma: &neon-unstable-fcma + FnCall: [unstable, ['feature = "stdarch_neon_fcma"', 'issue = "117222"']] + +intrinsics: + - name: "vand{neon_type.no}" + doc: Vector bitwise and + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vand]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [and]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + - int64x1_t + - int64x2_t + - uint64x1_t + - uint64x2_t + compose: + - FnCall: + - simd_and + - - a + - b + + - name: "vorr{neon_type.no}" + doc: "Vector bitwise or (immediate, inclusive)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vorr]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [orr]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + - int64x1_t + - int64x2_t + - uint64x1_t + - uint64x2_t + compose: + - FnCall: + - simd_or + - - a + - b + + - name: "veor{neon_type.no}" + doc: Vector bitwise exclusive or (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [veor]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [eor]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + - int64x1_t + - int64x2_t + - uint64x1_t + - uint64x2_t + compose: + - FnCall: + - simd_xor + - - a + - b + + - name: "vabd{neon_type.no}" + doc: Absolute difference between the arguments + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sabd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "sabd.{neon_type}" + links: + - link: "llvm.aarch64.neon.sabd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vabds.{neon_type}" + arch: arm + + - name: "vabd{neon_type.no}" + doc: Absolute difference between the arguments + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uabd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "uabd.{neon_type}" + links: + - link: "llvm.aarch64.neon.uabd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vabdu.{neon_type}" + arch: arm + + - name: "vabd{neon_type.no}" + doc: Absolute difference between the arguments of Floating + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.f32"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fabd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "fabd.{neon_type}" + links: + - link: "llvm.arm.neon.vabds.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.fabd.{neon_type}" + arch: aarch64,arm64ec + + - name: "vabdl{neon_type[0].noq}" + doc: Signed Absolute difference Long + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - target_feature + - - 'enable = "v7"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - '"vabdl.{neon_type[0]}"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - sabdl + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, uint8x8_t] + - [int16x4_t, int32x4_t, uint16x4_t] + - [int32x2_t, int64x2_t, uint32x2_t] + compose: + - Let: + - c + - "{neon_type[2]}" + - FnCall: + - simd_cast + - - FnCall: + - "vabd_{neon_type[0]}" + - - a + - b + - FnCall: + - simd_cast + - - c + + - name: "vceq{neon_type[0].no}" + doc: "Compare bitwise Equal (vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vceq{type[2]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmeq]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint8x8_t, ".i8"] + - [uint8x16_t, uint8x16_t, ".i8"] + - [int8x8_t, uint8x8_t, ".i8"] + - [int8x16_t, uint8x16_t, ".i8"] + - [poly8x8_t, uint8x8_t, ".i8"] + - [poly8x16_t, uint8x16_t, ".i8"] + - [uint16x4_t, uint16x4_t, ".i16"] + - [uint16x8_t, uint16x8_t, ".i16"] + - [int16x4_t, uint16x4_t, ".i16"] + - [int16x8_t, uint16x8_t, ".i16"] + - [uint32x2_t, uint32x2_t, ".i32"] + - [uint32x4_t, uint32x4_t, ".i32"] + - [int32x2_t, uint32x2_t, ".i32"] + - [int32x4_t, uint32x4_t, ".i32"] + compose: + - FnCall: [simd_eq, [a, b]] + + - name: "vceq{neon_type[0].no}" + doc: "Floating-point compare equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vceq.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmeq]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: [simd_eq, [a, b]] + + - name: "vtst{neon_type[0].no}" + doc: "Signed compare bitwise Test bits nonzero" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtst]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmtst]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] + - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] + - [poly8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [poly8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [poly16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] + - [poly16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + compose: + - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] + - Let: [d, "{type[2]}", "{type[3]}"] + - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] + + - name: "vabs{neon_type.no}" + doc: "Floating-point absolute value" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vabs]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fabs]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - FnCall: [simd_fabs, [a]] + + - name: "vcgt{neon_type[0].no}" + doc: "Compare signed greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{type[2]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, "s8"] + - [int8x16_t, uint8x16_t, "s8"] + - [int16x4_t, uint16x4_t, s16] + - [int16x8_t, uint16x8_t, s16] + - [int32x2_t, uint32x2_t, "s32"] + - [int32x4_t, uint32x4_t, "s32"] + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vcgt{neon_type.no}" + doc: "Compare unsigned greater than" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhi]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vcgt{neon_type[0].no}" + doc: "Floating-point compare greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: [simd_gt, [a, b]] + + - name: "vclt{neon_type[0].no}" + doc: "Compare signed less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type[0]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t] + - [int8x16_t, uint8x16_t] + - [int16x4_t, uint16x4_t] + - [int16x8_t, uint16x8_t] + - [int32x2_t, uint32x2_t] + - [int32x4_t, uint32x4_t] + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vcle{neon_type[0].no}" + doc: "Compare signed less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type[0]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t] + - [int8x16_t, uint8x16_t] + - [int16x4_t, uint16x4_t] + - [int16x8_t, uint16x8_t] + - [int32x2_t, uint32x2_t] + - [int32x4_t, uint32x4_t] + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vcle{neon_type[0].no}" + doc: "Floating-point compare less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vcge{neon_type[0].no}" + doc: "Compare signed greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type[0]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t] + - [int8x16_t, uint8x16_t] + - [int16x4_t, uint16x4_t] + - [int16x8_t, uint16x8_t] + - [int32x2_t, uint32x2_t] + - [int32x4_t, uint32x4_t] + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vcls{neon_type.no}" + doc: "Count leading sign bits" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcls.{neon_type}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "vcls{neon_type.no}" + links: + - link: "llvm.arm.neon.vcls.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.cls.{neon_type}" + arch: aarch64,arm64ec + + - name: "vcls{neon_type[0].no}" + doc: "Count leading sign bits" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcls]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [uint16x4_t, int16x4_t] + - [uint16x8_t, int16x8_t] + - [uint32x2_t, int32x2_t] + - [uint32x4_t, int32x4_t] + compose: + - FnCall: + - "vcls{neon_type[1].no}" + - - FnCall: [transmute, [a]] + + - name: "vclz{neon_type.no}" + doc: "Count leading zero bits" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz.i8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + compose: + - FnCall: ["vclz{neon_type.no}_", [a]] + + - name: "vclz{neon_type[0].no}" + doc: "Count leading zero bits" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz.i8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vclz{neon_type[1].no}_" + - - FnCall: [transmute, [a]] + + - name: "vclz{neon_type[0].no}" + doc: "Count leading zero bits" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz{type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, '.i16'] + - [int16x8_t, '.i16'] + - [int32x2_t, '.i32'] + - [int32x4_t, '.i32'] + compose: + - FnCall: ["vclz{neon_type[0].no}_", [a]] + + - name: "vclz{neon_type[0].no}" + doc: "Count leading zero bits" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz{type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint32x2_t, '.i32', int32x2_t] + - [uint32x4_t, '.i32', int32x4_t] + - [uint16x4_t, '.i16', int16x4_t] + - [uint16x8_t, '.i16', int16x8_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vclz{neon_type[2].no}_" + - - FnCall: [transmute, [a]] + + - name: "vcagt{neon_type[0].no}" + doc: "Floating-point absolute compare greater than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacgt.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - LLVMLink: + name: "vcagt{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vacgt.{neon_type[1]}.{neon_type[0]}" + arch: arm + - link: "llvm.aarch64.neon.facgt.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcage{neon_type[0].no}" + doc: "Floating-point absolute compare greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacge.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - LLVMLink: + name: "vcage{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vacge.{neon_type[1]}.{neon_type[0]}" + arch: arm + - link: "llvm.aarch64.neon.facge.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vcalt{neon_type[0].no}" + doc: "Floating-point absolute compare less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacgt.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: ["vcagt{neon_type[0].no}", [b, a]] + + - name: "vcale{neon_type[0].no}" + doc: "Floating-point absolute compare less than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacge.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: ["vcage{neon_type[0].no}", [b, a]] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [scvtf]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, float32x2_t] + - [int32x4_t, float32x4_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ucvtf]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint32x2_t, float32x2_t] + - [uint32x4_t, float32x4_t] + compose: + - FnCall: [simd_cast, [a]] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint32x2_t, float32x2_t] + - [uint32x4_t, float32x4_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.arm.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" + arch: arm + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, float32x2_t] + - [int32x4_t, float32x4_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.arm.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" + arch: arm + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, float32x2_t] + - [int32x4_t, float32x4_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] + + - name: "vcvt{neon_type[1].N}_{neon_type[0]}" + doc: "Fixed-point convert to floating-point" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint32x2_t, float32x2_t] + - [uint32x4_t, float32x4_t] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{neon_type[1].N}_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t, _n_s32_f32] + - [float32x4_t, int32x4_t, q_n_s32_f32] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.arm.neon.vcvtfp2fxs.{neon_type[1]}.{neon_type[0]}" + arch: arm + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, _n_u32_f32] + - [float32x4_t, uint32x4_t, q_n_u32_f32] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.arm.neon.vcvtfp2fxu.{neon_type[1]}.{neon_type[0]}" + arch: arm + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t, _n_s32_f32] + - [float32x4_t, int32x4_t, q_n_s32_f32] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: ["a: {type[0]}", "n: i32"] + links: + - link: "llvm.aarch64.neon.vcvtfp2fxs.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vcvt{type[2]}" + doc: "Floating-point convert to fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t, _n_u32_f32] + - [float32x4_t, uint32x4_t, q_n_u32_f32] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= 32']] + - LLVMLink: + name: "vcvt{type[2]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.vcvtfp2fxu.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vcvt{type[2]}", [a, N]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.8"', 'N = 4']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 4']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s8, int8x8_t, int8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_s8, int8x8_t, int8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_lane_u8, uint8x8_t, uint8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_u8, uint8x8_t, uint8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_lane_p8, poly8x8_t, poly8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_p8, poly8x8_t, poly8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.8"', 'N = 8']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 8']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [q_laneq_s8, int8x16_t, int8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_s8, int8x16_t, int8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [q_laneq_u8, uint8x16_t, uint8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_u8, uint8x16_t, uint8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [q_laneq_p8, poly8x16_t, poly8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_p8, poly8x16_t, poly8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.16"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_s16, int16x4_t, int16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_u16, uint16x4_t, uint16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_lane_p16, poly16x4_t, poly16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [q_lane_p16, poly16x4_t, poly16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.16"', 'N = 4']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 4']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_s16, int16x8_t, int16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] + - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_u16, uint16x8_t, uint16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] + - [q_laneq_p16, poly16x8_t, poly16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_p16, poly16x8_t, poly16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.32"', 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s32, int32x2_t, int32x2_t, '1', '[N as u32, N as u32]'] + - [q_lane_s32, int32x2_t, int32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] + - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[N as u32, N as u32]'] + - [q_lane_u32, uint32x2_t, uint32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] + - [_lane_f32, float32x2_t, float32x2_t, '1', '[N as u32, N as u32]'] + - [q_lane_f32, float32x2_t, float32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.32"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_s32, int32x4_t, int32x2_t, '2', '[N as u32, N as u32]'] + - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_u32, uint32x4_t, uint32x2_t, '2', '[N as u32, N as u32]'] + - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] + - [_laneq_f32, float32x4_t, float32x2_t, '2', '[N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [q_laneq_s64, int64x2_t, '1', '[N as u32, N as u32]'] + - [q_laneq_u64, uint64x2_t, '1', '[N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] + - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [q_lane_s64, int64x1_t, int64x2_t] + - [q_lane_u64, uint64x1_t, uint64x2_t] + compose: + - FnCall: [static_assert!, ['N == 0']] + - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'N = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'N = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s64, int64x1_t] + - [_lane_u64, uint64x1_t] + compose: + - FnCall: [static_assert!, ['N == 0']] + - Identifier: [a, Symbol] + + - name: "vdup{type[0]}" + doc: "Set all vector lanes to the same value" + arguments: ["a: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [_laneq_s64, int64x2_t, int64x1_t, '::'] + - [_laneq_u64, uint64x2_t, uint64x1_t, '::'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, 1]] + - FnCall: + - "transmute{type[3]}" + - - FnCall: [simd_extract!, [a, 'N as u32']] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 7']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 7']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + - [int16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + - [uint8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + - [uint16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + - [poly8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + - [poly16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 15']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 15']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] + - [uint8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] + - [poly8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 3']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 3']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, 'static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + - [int32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + - [uint16x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + - [uint32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + - [poly16x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + - [float32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + - [uint32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + - [float32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vext{neon_type[0].no}" + doc: "Extract vector from pair of vectors" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, 'static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + - [uint64x2_t, 'static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] + compose: + - Identifier: ["{type[1]}", Symbol] + + - name: "vmla{neon_type[0].no}" + doc: "Multiply-add to accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla{type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, ".i8"] + - [int8x16_t, ".i8"] + - [uint8x8_t, ".i8"] + - [uint8x16_t, ".i8"] + - [int16x4_t, ".i16"] + - [int16x8_t, ".i16"] + - [uint16x4_t, ".i16"] + - [uint16x8_t, ".i16"] + - [int32x2_t, ".i32"] + - [int32x4_t, ".i32"] + - [uint32x2_t, ".i32"] + - [uint32x4_t, ".i32"] + compose: + - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] + + - name: "vmla{neon_type.no}" + doc: "Floating-point multiply-add to accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] + + - name: "vmlal{neon_type[1].no}" + doc: "Signed multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[2]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, "s8"] + - [int32x4_t, int16x4_t, "s16"] + - [int64x2_t, int32x2_t, "s32"] + compose: + - FnCall: [simd_add, [a, {FnCall: ["vmull_{type[2]}", [b, c]]}]] + + - name: "vmlal_n_{type[4]}" + doc: "Vector widening multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[4]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, "i16", int32x4_t, 's16'] + - [int64x2_t, int32x2_t, "i32", int64x2_t, 's32'] + compose: + - FnCall: + - "vmlal{neon_type[1].noq}" + - - a + - b + - FnCall: ["vdup_n_{neon_type[1]}", [c]] + + - name: "vmlal_n_{type[2]}" + doc: "Vector widening multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[2]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x4_t, "u16", uint32x4_t] + - [uint64x2_t, uint32x2_t, "u32", uint64x2_t] + compose: + - FnCall: + - "vmlal{neon_type[1].noq}" + - - a + - b + - FnCall: ["vdup_n_{neon_type[1]}", [c]] + + - name: "vmlal_lane{neon_type[2].no}" + doc: "Vector widening multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int64x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [int64x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmlal_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] + + - name: "vmlal_lane{neon_type[2].no}" + doc: "Vector widening multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x4_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint16x4_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x2_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x2_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] + - FnCall: + - "vmlal_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, '{type[5]}']] + + - name: "vmlal_{neon_type[1]}" + doc: "Unsigned multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t] + - [uint32x4_t, uint16x4_t] + - [uint64x2_t, uint32x2_t] + compose: + - FnCall: + - simd_add + - - a + - FnCall: ["vmull_{neon_type[1]}", [b, c]] + + - name: "vmls{neon_type[0].no}" + doc: "Multiply-subtract from accumulator" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls{type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, '.i8'] + - [int8x16_t, '.i8'] + - [uint8x8_t, '.i8'] + - [uint8x16_t, '.i8'] + - [int16x4_t, ".i16"] + - [int16x8_t, ".i16"] + - [uint16x4_t, ".i16"] + - [uint16x8_t, ".i16"] + - [int32x2_t, ".i32"] + - [int32x4_t, ".i32"] + - [uint32x2_t, ".i32"] + - [uint32x4_t, ".i32"] + compose: + - FnCall: + - simd_sub + - - a + - FnCall: [simd_mul, [b, c]] + + - name: "vmlsl_{neon_type[1]}" + doc: "Signed multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t] + - [int32x4_t, int16x4_t] + - [int64x2_t, int32x2_t] + compose: + - FnCall: [simd_sub, [a, {FnCall: ["vmull_{neon_type[1]}", [b, c]]}]] + + - name: "vmlsl_n_{neon_type[1]}" + doc: "Vector widening multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, "i16"] + - [int64x2_t, int32x2_t, "i32"] + compose: + - FnCall: ["vmlsl_{neon_type[1]}", [a, b, {FnCall: ["vdup_n_{neon_type[1]}", [c]]}]] + + - name: "vmlsl_n_{neon_type[1]}" + doc: "Vector widening multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x4_t, "u16"] + - [uint64x2_t, uint32x2_t, "u32"] + compose: + - FnCall: ["vmlsl_{neon_type[1]}", [a, b, {FnCall: ["vdup_n_{neon_type[1]}", [c]]}]] + + - name: "vmlsl_lane{neon_type[2].no}" + doc: "Vector widening multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmlsl_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmlsl_lane{neon_type[2].no}" + doc: "Vector widening multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int32x2_t, int32x2_t, '[LANE as u32, LANE as u32]', '1'] + - [int64x2_t, int32x2_t, int32x4_t, '[LANE as u32, LANE as u32]', '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] + - FnCall: + - "vmlsl_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + + - name: "vmlsl_lane{neon_type[2].no}" + doc: "Vector widening multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint32x4_t, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x4_t, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [uint64x2_t, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmlsl_{neon_type[1]}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmlsl_{neon_type[1]}" + doc: "Unsigned multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t] + - [uint32x4_t, uint16x4_t] + - [uint64x2_t, uint32x2_t] + compose: + - FnCall: [simd_sub, [a, {FnCall: ["vmull_{neon_type[1]}", [b, c]]}]] + + - name: "vneg{neon_type[0].no}" + doc: Negate + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vneg.{type[1]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [neg]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, 's8'] + - [int8x16_t, 's8'] + - [int16x4_t, 's16'] + - [int16x8_t, 's16'] + - [int32x2_t, 's32'] + - [int32x4_t, 's32'] + compose: + - FnCall: [simd_neg, [a]] + + - name: "vneg{neon_type[0].no}" + doc: Negate + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vneg.{type[1]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fneg]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [float32x2_t, 'f32'] + - [float32x4_t, 'f32'] + compose: + - FnCall: [simd_neg, [a]] + + - name: "vqneg{neon_type[0].no}" + doc: Signed saturating negate + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqneg.{type[1]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqneg]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, 's8', 'i8'] + - [int8x16_t, 's8', 'i8'] + - [int16x4_t, 's16', 'i16'] + - [int16x8_t, 's16', 'i16'] + - [int32x2_t, 's32', 'i32'] + - [int32x4_t, 's32', 'i32'] + compose: + - LLVMLink: + name: "sqneg.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.sqneg.v{neon_type[0].lane}{type[2]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vqneg.v{neon_type[0].lane}{type[2]}" + arch: arm + + - name: "vqsub{neon_type[0].no}" + doc: Saturating subtract + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqsub.{type[1]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uqsub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint8x8_t, u8, i8] + - [uint8x16_t, u8, i8] + - [uint16x4_t, u16, i16] + - [uint16x8_t, u16, i16] + - [uint32x2_t, u32, i32] + - [uint32x4_t, u32, i32] + - [uint64x1_t, u64, i64] + - [uint64x2_t, u64, i64] + compose: + - LLVMLink: + name: "uqsub.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.uqsub.v{neon_type[0].lane}{type[2]}" + arch: aarch64,arm64ec + - link: "llvm.usub.sat.{neon_type[0].lane}{type[2]}" + arch: arm + + - name: "vqsub{neon_type[0].no}" + doc: Saturating subtract + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqsub.{type[1]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqsub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, s8, i8] + - [int8x16_t, s8, i8] + - [int16x4_t, s16, i16] + - [int16x8_t, s16, i16] + - [int32x2_t, s32, i32] + - [int32x4_t, s32, i32] + - [int64x1_t, s64, i64] + - [int64x2_t, s64, i64] + compose: + - LLVMLink: + name: "sqsub.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.sqsub.v{neon_type[0].lane}{type[2]}" + arch: aarch64,arm64ec + - link: "llvm.ssub.sat.{neon_type[0].lane}{type[2]}" + arch: arm + + - name: "vhadd{neon_type.no}" + doc: Halving add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - target_feature + - - 'enable = "v7"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - '"vhadd.{neon_type}"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - uhadd + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "uhadd.{neon_type}" + links: + - link: "llvm.aarch64.neon.uhadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vhaddu.{neon_type}" + arch: arm + + - name: "vhadd{neon_type.no}" + doc: Halving add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - target_feature + - - 'enable = "v7"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - '"vhadd.{neon_type}"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - shadd + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "shadd.{neon_type}" + links: + - link: "llvm.aarch64.neon.shadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vhadds.{neon_type}" + arch: arm + + - name: "vrhadd{neon_type.no}" + doc: Rounding halving add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vrhadd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [srhadd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "vrhadd.{neon_type}" + links: + - link: "llvm.aarch64.neon.srhadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vrhadds.{neon_type}" + arch: arm + + - name: "vrhadd{neon_type.no}" + doc: Rounding halving add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vrhadd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [urhadd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "vrhaddu.{neon_type}" + links: + - link: "llvm.aarch64.neon.urhadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vrhaddu.{neon_type}" + arch: arm + + - name: "vrndn{neon_type.no}" + doc: "Floating-point round to integral, to nearest with ties to even" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vrintn]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frintn]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "llvm.frinn.{neon_type}" + links: + - link: "llvm.aarch64.neon.frintn.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vrintn.{neon_type}" + arch: arm + + - name: "vqadd{neon_type.no}" + doc: Saturating add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqadd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uqadd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + - uint64x1_t + - uint64x2_t + compose: + - LLVMLink: + name: "uqadd.{neon_type}" + links: + - link: "llvm.aarch64.neon.uqadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.uadd.sat.{neon_type}" + arch: arm + + - name: "vqadd{neon_type.no}" + doc: Saturating add + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqadd.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqadd]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "sqadd.{neon_type}" + links: + - link: "llvm.aarch64.neon.sqadd.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.sadd.sat.{neon_type}" + arch: arm + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const f32", float32x2x2_t] + - ["*const f32", float32x4x2_t] + - ["*const f32", float32x2x3_t] + - ["*const f32", float32x4x3_t] + - ["*const f32", float32x2x4_t] + - ["*const f32", float32x4x4_t] + compose: + - LLVMLink: + name: "vld1x{neon_type[1].tuple}.{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f{neon_type[1].base}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f{neon_type[1].base}" + arch: arm + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t] + - ["*const i8", int8x16x2_t] + - ["*const i8", int8x8x3_t] + - ["*const i8", int8x16x3_t] + - ["*const i8", int8x8x4_t] + - ["*const i8", int8x16x4_t] + - ["*const i16", int16x4x2_t] + - ["*const i16", int16x8x2_t] + - ["*const i16", int16x4x3_t] + - ["*const i16", int16x8x3_t] + - ["*const i16", int16x4x4_t] + - ["*const i16", int16x8x4_t] + - ["*const i32", int32x2x2_t] + - ["*const i32", int32x4x2_t] + - ["*const i32", int32x2x3_t] + - ["*const i32", int32x4x3_t] + - ["*const i32", int32x2x4_t] + - ["*const i32", int32x4x4_t] + - ["*const i64", int64x1x2_t] + - ["*const i64", int64x1x3_t] + - ["*const i64", int64x1x4_t] + - ["*const i64", int64x2x2_t] + - ["*const i64", int64x2x3_t] + - ["*const i64", int64x2x4_t] + compose: + - LLVMLink: + name: "ld1x{neon_type[1].tuple}.{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}i{neon_type[1].base}.p0i{neon_type[1].base}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vld1x{neon_type[1].tuple}.v{neon_type[1].lane}i{neon_type[1].base}.p0i{neon_type[1].base}" + arch: arm + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x8x2_t, int8x8x2_t] + - ["*const u8", uint8x16x2_t, int8x16x2_t] + - ["*const u8", uint8x8x3_t, int8x8x3_t] + - ["*const u8", uint8x16x3_t, int8x16x3_t] + - ["*const u8", uint8x8x4_t, int8x8x4_t] + - ["*const u8", uint8x16x4_t, int8x16x4_t] + - ["*const u16", uint16x4x2_t, int16x4x2_t] + - ["*const u16", uint16x8x2_t, int16x8x2_t] + - ["*const u16", uint16x4x3_t, int16x4x3_t] + - ["*const u16", uint16x8x3_t, int16x8x3_t] + - ["*const u16", uint16x4x4_t, int16x4x4_t] + - ["*const u16", uint16x8x4_t, int16x8x4_t] + - ["*const u32", uint32x2x2_t, int32x2x2_t] + - ["*const u32", uint32x4x2_t, int32x4x2_t] + - ["*const u32", uint32x2x3_t, int32x2x3_t] + - ["*const u32", uint32x4x3_t, int32x4x3_t] + - ["*const u32", uint32x2x4_t, int32x2x4_t] + - ["*const u32", uint32x4x4_t, int32x4x4_t] + - ["*const u64", uint64x1x2_t, int64x1x2_t] + - ["*const u64", uint64x1x3_t, int64x1x3_t] + - ["*const u64", uint64x1x4_t, int64x1x4_t] + - ["*const u64", uint64x2x2_t, int64x2x2_t] + - ["*const u64", uint64x2x3_t, int64x2x3_t] + - ["*const u64", uint64x2x4_t, int64x2x4_t] + - ["*const p8", poly8x8x2_t, int8x8x2_t] + - ["*const p8", poly8x8x3_t, int8x8x3_t] + - ["*const p8", poly8x8x4_t, int8x8x4_t] + - ["*const p8", poly8x16x2_t, int8x16x2_t] + - ["*const p8", poly8x16x3_t, int8x16x3_t] + - ["*const p8", poly8x16x4_t, int8x16x4_t] + - ["*const p16", poly16x4x2_t, int16x4x2_t] + - ["*const p16", poly16x4x3_t, int16x4x3_t] + - ["*const p16", poly16x4x4_t, int16x4x4_t] + - ["*const p16", poly16x8x2_t, int16x8x2_t] + - ["*const p16", poly16x8x3_t, int16x8x3_t] + - ["*const p16", poly16x8x4_t, int16x8x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld1{neon_type[2].no}" + - - FnCall: + - transmute + - - a + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x3_t, int64x1x3_t] + - ["*const p64", poly64x1x4_t, int64x1x4_t] + - ["*const p64", poly64x2x2_t, int64x2x2_t] + - ["*const p64", poly64x2x3_t, int64x2x3_t] + - ["*const p64", poly64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld1{neon_type[2].no}" + - - FnCall: + - transmute + - - a + + - name: "vld1{neon_type[1].no}" + doc: "Load multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld1{neon_type[2].no}" + - - FnCall: + - transmute + - - a + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [vld2] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8] + - ["*const i16", int16x4x2_t, i16] + - ["*const i32", int32x2x2_t, i32] + - ["*const i8", int8x16x2_t, i8] + - ["*const i16", int16x8x2_t, i16] + - ["*const i32", int32x4x2_t, i32] + - ["*const f32", float32x2x2_t, f32] + - ["*const f32", float32x4x2_t, f32] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as *const i8" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x2_t, i64] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as *const i8" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - *neon-stable + assert_instr: [ld2] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8, int8x8_t] + - ["*const i16", int16x4x2_t, i16, int16x4_t] + - ["*const i32", int32x2x2_t, i32, int32x2_t] + - ["*const i8", int8x16x2_t, i8, int8x16_t] + - ["*const i16", int16x8x2_t, i16, int16x8_t] + - ["*const i32", int32x4x2_t, i32, int32x4_t] + - ["*const f32", float32x2x2_t, f32, float32x2_t] + - ["*const f32", float32x4x2_t, f32, float32x4_t] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const {neon_type[3]}" + links: + - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as _" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - *neon-stable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x2_t, i64, int64x1_t] + compose: + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const {neon_type[3]}" + links: + - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].nox}" + - - "a as _" + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x8x2_t, int8x8x2_t] + - ["*const u16", uint16x4x2_t, int16x4x2_t] + - ["*const u32", uint32x2x2_t, int32x2x2_t] + - ["*const u8", uint8x16x2_t, int8x16x2_t] + - ["*const u16", uint16x8x2_t, int16x8x2_t] + - ["*const u32", uint32x4x2_t, int32x4x2_t] + - ["*const p8", poly8x8x2_t, int8x8x2_t] + - ["*const p16", poly16x4x2_t, int16x4x2_t] + - ["*const p8", poly8x16x2_t, int8x16x2_t] + - ["*const p16", poly16x8x2_t, int16x8x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + + - name: "vld2{neon_type[1].nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - FnCall: + - cfg_attr + - - test + - FnCall: + - assert_instr + - - vld2 + - "LANE = 0" + - FnCall: + - rustc_legacy_const_generics + - - "2" + - *neon-unstable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8, int8x8_t, "3"] + - ["*const i16", int16x4x2_t, i16, int16x4_t, "2"] + - ["*const i32", int32x2x2_t, i32, int32x2_t, "1"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[4]}" + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "a: {neon_type[3]}" + - "b: {neon_type[3]}" + - "n: i32" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2lane.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2_lane{neon_type[1].nox}" + - - "a as _" + - "b.0" + - "b.1" + - "LANE" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x8x2_t, int8x8_t, "3"] + - ["*const u16", uint16x4x2_t, int16x4_t, "2"] + - ["*const u32", uint32x2x2_t, int32x2_t, "1"] + - ["*const u16", uint16x8x2_t, int16x8_t, "3"] + - ["*const u32", uint32x4x2_t, int32x4_t, "2"] + - ["*const p8", poly8x8x2_t, int8x8_t, "3"] + - ["*const p16", poly16x4x2_t, int16x4_t, "2"] + - ["*const p16", poly16x8x2_t, int16x8_t, "3"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[3]}" + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - FnCall: + - cfg_attr + - - test + - FnCall: + - assert_instr + - - ld2 + - "LANE = 0" + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8, int8x8_t, "3"] + - ["*const i16", int16x4x2_t, i16, int16x4_t, "2"] + - ["*const i32", int32x2x2_t, i32, int32x2_t, "1"] + - ["*const i16", int16x8x2_t, i16, int16x8_t, "3"] + - ["*const i32", int32x4x2_t, i32, int32x4_t, "2"] + - ["*const f32", float32x2x2_t, f32, float32x2_t, "2"] + - ["*const f32", float32x4x2_t, f32, float32x4_t, "2"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[4]}" + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "a: {neon_type[3]}" + - "b: {neon_type[3]}" + - "n: i64" + - "ptr: *const i8" + links: + - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].lane_nox}" + - - "b.0" + - "b.1" + - "LANE as i64" + - "a as _" + + - name: "vld2{neon_type[1].lane_nox}" + doc: Load multiple 2-element structures to two registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - FnCall: + - cfg_attr + - - test + - FnCall: + - assert_instr + - - vld2 + - "LANE = 0" + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-unstable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ["*const i16", int16x8x2_t, i16, int16x8_t, "3"] + - ["*const i32", int32x4x2_t, i32, int32x4_t, "2"] + - ["*const f32", float32x2x2_t, f32, float32x2_t, "1"] + - ["*const f32", float32x4x2_t, f32, float32x4_t, "2"] + compose: + - FnCall: + - "static_assert_uimm_bits!" + - - LANE + - "{type[4]}" + - LLVMLink: + name: "vld2.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "a: {neon_type[3]}" + - "b: {neon_type[3]}" + - "n: i32" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2lane.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2{neon_type[1].lane_nox}" + - - "a as _" + - "b.0" + - "b.1" + - "LANE" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x2_t, i64] + compose: + - LLVMLink: + name: "vld2dup.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2dup.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2{neon_type[1].dup_nox}" + - - "a as *const i8" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - *neon-stable + assert_instr: [ld2r] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x2_t, i64] + compose: + - LLVMLink: + name: "vld2dup.{neon_type[1]}" + arguments: + - "ptr: *const i64" + links: + - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0i64" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].dup_nox}" + - - "a as _" + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [vld2] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8] + - ["*const i16", int16x4x2_t, i16] + - ["*const i32", int32x2x2_t, i32] + - ["*const i8", int8x16x2_t, i8] + - ["*const i16", int16x8x2_t, i16] + - ["*const i32", int32x4x2_t, i32] + - ["*const f32", float32x2x2_t, f32] + - ["*const f32", float32x4x2_t, f32] + compose: + - LLVMLink: + name: "vld2dup.{neon_type[1]}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld2dup.v{neon_type[1].lane}{type[2]}.p0i8" + arch: arm + - FnCall: + - "_vld2{neon_type[1].dup_nox}" + - - "a as *const i8" + - "{neon_type[1].base_byte_size}" + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x8x2_t, int8x8x2_t] + - ["*const u16", uint16x4x2_t, int16x4x2_t] + - ["*const u32", uint32x2x2_t, int32x2x2_t] + - ["*const u8", uint8x16x2_t, int8x16x2_t] + - ["*const u16", uint16x8x2_t, int16x8x2_t] + - ["*const u32", uint32x4x2_t, int32x4x2_t] + - ["*const p8", poly8x8x2_t, int8x8x2_t] + - ["*const p16", poly16x4x2_t, int16x4x2_t] + - ["*const p8", poly8x16x2_t, int8x16x2_t] + - ["*const p16", poly16x8x2_t, int16x8x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - transmute + - - FnCall: + - "vld2{neon_type[2].dup_nox}" + - - FnCall: + - transmute + - - a + + - name: "vld2{neon_type[1].dup_nox}" + doc: Load single 2-element structure and replicate to all lanes of two registers + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - FnCall: + - cfg + - - FnCall: + - not + - - 'target_arch = "arm"' + - *neon-stable + assert_instr: [ld2r] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x2_t, i8] + - ["*const i16", int16x4x2_t, i16] + - ["*const i32", int32x2x2_t, i32] + - ["*const i8", int8x16x2_t, i8] + - ["*const i16", int16x8x2_t, i16] + - ["*const i32", int32x4x2_t, i32] + - ["*const f32", float32x2x2_t, f32] + - ["*const f32", float32x4x2_t, f32] + compose: + - LLVMLink: + name: "vld2dup.{neon_type[1]}" + arguments: + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0{type[2]}" + arch: aarch64,arm64ec + - FnCall: + - "_vld2{neon_type[1].dup_nox}" + - - "a as _" + + - name: "vld3{neon_type[1].lane_nox}" + doc: "Load multiple 3-element structures to two registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x3_t, int8x8_t, i8, '3'] + - ['*const i16', int16x8x3_t, int16x8_t, i16, '4'] + - ['*const i32', int32x4x3_t, int32x4_t, i32, '2'] + - ['*const i16', int16x4x3_t, int16x4_t, i16, '2'] + - ['*const i32', int32x2x3_t, int32x2_t, i32, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - LLVMLink: + name: 'ld3lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-stable + - *target-not-arm + assert_instr: [ld3] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x3_t, '*const int8x8_t', i8] + - ['*const i16', int16x4x3_t, '*const int16x4_t', i16] + - ['*const i32', int32x2x3_t, '*const int32x2_t', i32] + - ['*const i8', int8x16x3_t, '*const int8x16_t', i8] + - ['*const i16', int16x8x3_t, '*const int16x8_t', i16] + - ['*const i32', int32x4x3_t, '*const int32x4_t', i32] + - ['*const f32', float32x2x3_t, '*const float32x2_t', f32] + - ['*const f32', float32x4x3_t, '*const float32x4_t', f32] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].nox}' + arguments: + - 'ptr: {type[2]}' + links: + - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-stable + - *target-not-arm + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ['*const i64', int64x1x3_t, '*const int64x1_t', i64] + compose: + - LLVMLink: + name: "vld3{neon_type[1].nox}" + arguments: + - 'ptr: {type[2]}' + links: + - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vld3] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x3_t, i8] + - ['*const i16', int16x4x3_t, i16] + - ['*const i32', int32x2x3_t, i32] + - ['*const i8', int8x16x3_t, i8] + - ['*const i16', int16x8x3_t, i16] + - ['*const i32', int32x4x3_t, i32] + - ['*const f32', float32x2x3_t, f32] + - ['*const f32', float32x4x3_t, f32] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].nox}', ['a as *const i8', '{neon_type[1].base_byte_size}']] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [nop] + types: + - ['*const i64', int64x1x3_t, i64] + safety: + unsafe: [neon] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].nox}', ['a as *const i8', '{neon_type[1].base_byte_size}']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ['*const f32', float32x4x3_t, float32x4_t, f32, '2'] + - ['*const f32', float32x2x3_t, float32x2_t, f32, '1'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] + - LLVMLink: + name: 'vld3{neon_type[1].lane_nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vld3{neon_type[2].lane_nox}" + doc: "Load multiple 3-element structures to three registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ['*const f32', float32x2x3_t, float32x2_t, f32, '1', '4'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] + - LLVMLink: + name: 'vld3{neon_type[1].lane_nox}' + arguments: + - 'ptr: *const i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] + + - name: "vld3{neon_type[2].lane_nox}" + doc: "Load multiple 3-element structures to two registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x3_t, int8x8_t, i8, '3', '1'] + - ['*const i16', int16x4x3_t, int16x4_t, i16, '2', '2'] + - ['*const i32', int32x2x3_t, int32x2_t, i32, '1', '4'] + - ['*const i16', int16x8x3_t, int16x8_t, i16, '3', '2'] + - ['*const i32', int32x4x3_t, int32x4_t, i32, '2', '4'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] + - LLVMLink: + name: 'vld3{neon_type[1].lane_nox}' + arguments: + - 'ptr: *const i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] + + - name: "vld3{neon_type[2].lane_nox}" + doc: "Load multiple 3-element structures to three registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ['*const f32', float32x4x3_t, float32x4_t, f32, '2', '4'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] + - LLVMLink: + name: 'vld3{neon_type[1].lane_nox}' + arguments: + - 'ptr: *const i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] + + - name: "vld3{neon_type[1].lane_nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ['*const u8', uint8x8x3_t, int8x8x3_t, '3'] + - ['*const u16', uint16x4x3_t, int16x4x3_t, '2'] + - ['*const u32', uint32x2x3_t, int32x2x3_t, '1'] + - ['*const p8', poly8x8x3_t, int8x8x3_t, '3'] + - ['*const u16', uint16x8x3_t, int16x8x3_t, '3'] + - ['*const p16', poly16x4x3_t, int16x4x3_t, '2'] + - ['*const p16', poly16x8x3_t, int16x8x3_t, '3'] + - ['*const u32', uint32x4x3_t, int32x4x3_t, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const u8', uint8x8x3_t, int8x8x3_t] + - ['*const u8', uint8x16x3_t, int8x16x3_t] + - ['*const u16', uint16x4x3_t, int16x4x3_t] + - ['*const u32', uint32x2x3_t, int32x2x3_t] + - ['*const u16', uint16x8x3_t, int16x8x3_t] + - ['*const u32', uint32x4x3_t, int32x4x3_t] + - ['*const p8', poly8x8x3_t, int8x8x3_t] + - ['*const p8', poly8x16x3_t, int8x16x3_t] + - ['*const p16', poly16x4x3_t, int16x4x3_t] + - ['*const p16', poly16x8x3_t, int16x8x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].nox}' + - - FnCall: [transmute, [a]] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const u64', uint64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].nox}' + - - FnCall: [transmute, [a]] + + - name: "vld3{neon_type[1].nox}" + doc: Load multiple 3-element structures to three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*target-not-arm, *neon-stable] + assert_instr: [ld3r] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x3_t, i8] + - ["*const i16", int16x4x3_t, i16] + - ["*const i32", int32x2x3_t, i32] + - ["*const i32", int32x4x3_t, i32] + - ["*const i16", int16x8x3_t, i16] + - ["*const i8", int8x16x3_t, i8] + - ["*const i64", int64x1x3_t, i64] + - ["*const f32", float32x4x3_t, f32] + - ["*const f32", float32x2x3_t, f32] + compose: + - LLVMLink: + name: 'ld3r{neon_type[1].dup_nox}' + arguments: + - 'ptr: {type[0]}' + links: + - link: 'llvm.aarch64.neon.ld3r.v{neon_type[1].lane}{type[2]}.p0{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as _']] + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*enable-v7, *target-is-arm, *neon-unstable] + assert_instr: [vld3] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x3_t, i8, '1'] + - ["*const i16", int16x4x3_t, i16, '2'] + - ["*const i32", int32x2x3_t, i32, '4'] + - ["*const i8", int8x16x3_t, i8, '1'] + - ["*const i16", int16x8x3_t, i16, '2'] + - ["*const i32", int32x4x3_t, i32, '4'] + - ["*const f32", float32x4x3_t, f32, '4'] + - ["*const f32", float32x2x3_t, f32, '4'] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].dup_nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3dup.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as *const i8', '{type[3]}']] + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const u8', uint8x8x3_t, int8x8x3_t] + - ['*const u16', uint16x4x3_t, int16x4x3_t] + - ['*const u32', uint32x2x3_t, int32x2x3_t] + - ['*const u8', uint8x16x3_t, int8x16x3_t] + - ['*const u16', uint16x8x3_t, int16x8x3_t] + - ['*const u32', uint32x4x3_t, int32x4x3_t] + - ['*const p8', poly8x8x3_t, int8x8x3_t] + - ['*const p16', poly16x4x3_t, int16x4x3_t] + - ['*const p8', poly8x16x3_t, int8x16x3_t] + - ['*const p16', poly16x8x3_t, int16x8x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].dup_nox}' + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*target-is-arm, *enable-v7, *neon-unstable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x3_t, i64, '8'] + compose: + - LLVMLink: + name: 'vld3{neon_type[1].dup_nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld3dup.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as *const i8', '{type[3]}']] + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].dup_nox}' + - - FnCall: + - transmute + - - a + + - name: "vld3{neon_type[1].dup_nox}" + doc: Load single 3-element structure and replicate to all lanes of three registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld3{neon_type[2].dup_nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - *neon-stable + assert_instr: [ld4] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x4_t, i8, '*const int8x8_t'] + - ['*const i32', int32x4x4_t, i32, '*const int32x4_t'] + - ['*const i16', int16x4x4_t, i16, '*const int16x4_t'] + - ['*const i32', int32x2x4_t, i32, '*const int32x2_t'] + - ['*const i8', int8x16x4_t, i8, '*const int8x16_t'] + - ['*const i16', int16x8x4_t, i16, '*const int16x8_t'] + - ['*const f32', float32x2x4_t, f32, '*const float32x2_t'] + - ['*const f32', float32x4x4_t, f32, '*const float32x4_t'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: {type[3]}' + links: + - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: [*target-not-arm, *neon-stable] + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ['*const i64', int64x1x4_t, i64, '*const int64x1_t'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: {type[3]}' + links: + - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable + static_defs: + - "const LANE: i32" + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x4_t, int8x8_t, i8, '3'] + - ['*const i16', int16x4x4_t, int16x4_t, i16, '2'] + - ['*const i16', int16x8x4_t, int16x8_t, i16, '3'] + - ['*const i32', int32x2x4_t, int32x2_t, i32, '1'] + - ['*const i32', int32x4x4_t, int32x4_t, i32, '2'] + - ['*const f32', float32x2x4_t, float32x2_t, f32, '1'] + - ['*const f32', float32x4x4_t, float32x4_t, f32, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] + - LLVMLink: + name: 'ld4lane.{neon_type[2]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i64' + - 'ptr: *const i8' + links: + - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [vld4] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x4_t, i8, '1'] + - ['*const i16', int16x4x4_t, i16, '2'] + - ['*const i32', int32x2x4_t, i32, '4'] + - ['*const i8', int8x16x4_t, i8, '1'] + - ['*const i16', int16x8x4_t, i16, '2'] + - ['*const i32', int32x4x4_t, i32, '4'] + - ['*const f32', float32x4x4_t, f32, '4'] + - ['*const f32', float32x2x4_t, f32, '4'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld4.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld4{neon_type[1].nox}', ['a as *const i8', '{type[3]}']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - ['*const i64', int64x1x4_t, i64, '8'] + compose: + - LLVMLink: + name: 'vld4{neon_type[1].nox}' + arguments: + - 'ptr: *const i8' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld4.v{neon_type[1].lane}{type[2]}.p0i8' + arch: arm + - FnCall: ['_vld4{neon_type[1].nox}', ['a as *const i8', '{type[3]}']] + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const u8', uint8x8x4_t, int8x8x4_t] + - ['*const u16', uint16x4x4_t, int16x4x4_t] + - ['*const u32', uint32x2x4_t, int32x2x4_t] + - ['*const u8', uint8x16x4_t, int8x16x4_t] + - ['*const u16', uint16x8x4_t, int16x8x4_t] + - ['*const u32', uint32x4x4_t, int32x4x4_t] + - ['*const p8', poly8x8x4_t, int8x8x4_t] + - ['*const p16', poly16x4x4_t, int16x4x4_t] + - ['*const p8', poly8x16x4_t, int8x16x4_t] + - ['*const p16', poly16x8x4_t, int16x8x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const u64', uint64x1x4_t, int64x1x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v8 + - FnCall: + - target_feature + - - 'enable = "neon,aes"' + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['*const p64', poly64x1x4_t, int64x1x4_t] + compose: + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].nox}' + - - FnCall: + - transmute + - - a + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *enable-v7 + - *target-is-arm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-unstable + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['*const i8', int8x8x4_t, int8x8_t, i8, '1', '3'] + - ['*const i16', int16x4x4_t, int16x4_t, i16, '2', '2'] + - ['*const i32', int32x2x4_t, int32x2_t, i32, '4', '1'] + - ['*const i16', int16x8x4_t, int16x8_t, i16, '2', '3'] + - ['*const i32', int32x4x4_t, int32x4_t, i32, '4', '2'] + - ['*const f32', float32x2x4_t, float32x2_t, f32, '4', '1'] + - ['*const f32', float32x4x4_t, float32x4_t, f32, '4', '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[5]}']] + - LLVMLink: + name: 'ld4lane.{neon_type[2]}' + arguments: + - 'ptr: *const i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vld4lane.v{neon_type[1].lane}{type[3]}.p0i8' + arch: arm + - FnCall: ['_vld4{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', LANE, '{type[4]}']] + + - name: "vld4{neon_type[1].lane_nox}" + doc: Load multiple 4-element structures to four registers + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['*const u8', uint8x8x4_t, int8x8x4_t, '3'] + - ['*const u16', uint16x4x4_t, int16x4x4_t, '2'] + - ['*const u32', uint32x2x4_t, int32x2x4_t, '1'] + - ['*const u16', uint16x8x4_t, int16x8x4_t, '3'] + - ['*const u32', uint32x4x4_t, int32x4x4_t, '2'] + - ['*const p8', poly8x8x4_t, int8x8x4_t, '3'] + - ['*const p16', poly16x4x4_t, int16x4x4_t, '2'] + - ['*const p16', poly16x8x4_t, int16x8x4_t, '3'] + compose: + - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] + - FnCall: + - transmute + - - FnCall: + - 'vld4{neon_type[2].lane_nox}::' + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + types: + - ['*mut i64', int64x1_t] + - ['*mut u64', uint64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - *neon-v8 + - FnCall: + - target_feature + - - 'enable = "neon,aes"' + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + types: + - ['*mut p64', poly64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - *neon-v8 + - FnCall: + - target_feature + - - 'enable = "neon,aes"' + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + types: + - ['*mut p64', poly64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: "vst1{neon_type[1].lane_nox}" + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ["2"]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + types: + - ['*mut i8', int8x8_t, '3'] + - ['*mut i16', int16x4_t, '2'] + - ['*mut i32', int32x2_t, '1'] + - ['*mut i8', int8x16_t, '4'] + - ['*mut i16', int16x8_t, '3'] + - ['*mut i32', int32x4_t, '2'] + - ['*mut i64', int64x2_t, '1'] + - ['*mut u8', uint8x8_t, '3'] + - ['*mut u16', uint16x4_t, '2'] + - ['*mut u32', uint32x2_t, '1'] + - ['*mut u8', uint8x16_t, '4'] + - ['*mut u16', uint16x8_t, '3'] + - ['*mut u32', uint32x4_t, '2'] + - ['*mut u64', uint64x2_t, '1'] + - ['*mut p8', poly8x8_t, '3'] + - ['*mut p16', poly16x4_t, '2'] + - ['*mut p8', poly8x16_t, '4'] + - ['*mut p16', poly16x8_t, '3'] + - ['*mut f32', float32x2_t, '1'] + - ['*mut f32', float32x4_t, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - Assign: + - "*a" + - FnCall: [simd_extract!, [b, 'LANE as u32']] + - Identifier: [';', Symbol] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [st1] + types: + - [i8, int8x8x2_t, int8x8_t] + - [i16, int16x4x2_t, int16x4_t] + - [i32, int32x2x2_t, int32x2_t] + - [i64, int64x1x2_t, int64x1_t] + - [i8, int8x16x2_t, int8x16_t] + - [i16, int16x8x2_t, int16x8_t] + - [i32, int32x4x2_t, int32x4_t] + - [i64, int64x2x2_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'ptr: *mut {type[0]}' + links: + - link: 'llvm.aarch64.neon.st1x2.v{neon_type[1].lane}{type[0]}.p0{type[0]}' + arch: aarch64,arm64ec + - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'a']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [st1] + types: + - [i8, int8x8x3_t, int8x8_t] + - [i16, int16x4x3_t, int16x4_t] + - [i32, int32x2x3_t, int32x2_t] + - [i64, int64x1x3_t, int64x1_t] + - [i8, int8x16x3_t, int8x16_t] + - [i16, int16x8x3_t, int16x8_t] + - [i32, int32x4x3_t, int32x4_t] + - [i64, int64x2x3_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x3.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'ptr: *mut {type[0]}' + links: + - link: 'llvm.aarch64.neon.st1x3.v{neon_type[1].lane}{type[0]}.p0{type[0]}' + arch: aarch64,arm64ec + - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'b.2', 'a']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [st1] + types: + - [i8, int8x8x4_t, int8x8_t] + - [i16, int16x4x4_t, int16x4_t] + - [i32, int32x2x4_t, int32x2_t] + - [i64, int64x1x4_t, int64x1_t] + - [i8, int8x16x4_t, int8x16_t] + - [i16, int16x8x4_t, int16x8_t] + - [i32, int32x4x4_t, int32x4_t] + - [i64, int64x2x4_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x4.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'ptr: *mut {type[0]}' + links: + - link: 'llvm.aarch64.neon.st1x4.v{neon_type[1].lane}{type[0]}.p0{type[0]}' + arch: aarch64,arm64ec + - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'b.2', 'b.3', 'a']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [vst1] + types: + - [i8, int8x8x2_t, int8x8_t] + - [i16, int16x4x2_t, int16x4_t] + - [i32, int32x2x2_t, int32x2_t] + - [i64, int64x1x2_t, int64x1_t] + - [i8, int8x16x2_t, int8x16_t] + - [i16, int16x8x2_t, int16x8_t] + - [i32, int32x4x2_t, int32x4_t] + - [i64, int64x2x2_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x2.{neon_type[1]}' + arguments: + - 'ptr: *mut {type[0]}' + - 'a: {type[2]}' + - 'b: {type[2]}' + links: + - link: 'llvm.arm.neon.vst1x2.p0{type[0]}.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *enable-v7 + - *target-is-arm + - *neon-unstable + assert_instr: [vst1] + types: + - [i8, int8x8x3_t, int8x8_t] + - [i16, int16x4x3_t, int16x4_t] + - [i32, int32x2x3_t, int32x2_t] + - [i64, int64x1x3_t, int64x1_t] + - [i8, int8x16x3_t, int8x16_t] + - [i16, int16x8x3_t, int16x8_t] + - [i32, int32x4x3_t, int32x4_t] + - [i64, int64x2x3_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x3.{neon_type[1]}' + arguments: + - 'ptr: *mut {type[0]}' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + links: + - link: 'llvm.arm.neon.vst1x3.p0{type[0]}.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures from one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vst1] + types: + - [i8, int8x8x4_t, int8x8_t] + - [i16, int16x4x4_t, int16x4_t] + - [i32, int32x2x4_t, int32x2_t] + - [i64, int64x1x4_t, int64x1_t] + - [i8, int8x16x4_t, int8x16_t] + - [i16, int16x8x4_t, int16x8_t] + - [i32, int32x4x4_t, int32x4_t] + - [i64, int64x2x4_t, int64x2_t] + compose: + - LLVMLink: + name: 'st1x4.{neon_type[1]}' + arguments: + - 'ptr: *mut {type[0]}' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + links: + - link: 'llvm.arm.neon.vst1x4.p0{type[0]}.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2', 'b.3']] + + - name: 'vst1{neon_type[1].no}' + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + safety: + unsafe: [neon] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vst1] + types: + - [f32, float32x2x4_t, float32x2_t] + - [f32, float32x4x4_t, float32x4_t] + compose: + - LLVMLink: + name: 'st1x4.{neon_type[1]}' + arguments: + - 'ptr: *mut {type[0]}' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + links: + - link: 'llvm.arm.neon.vst1x4.p0{type[0]}.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2', 'b.3']] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v8 + - *neon-aes + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [p64, poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - "vst2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x2_t, int64x1_t] + compose: + - LLVMLink: + name: 'vst2.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst2.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst2{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', '8']] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u64, uint64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - "vst2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x2_t, int64x1_t] + compose: + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [st2] + safety: + unsafe: [neon] + types: + - [i8, int8x8x2_t, int8x8_t] + - [i16, int16x4x2_t, int16x4_t] + - [i32, int32x2x2_t, int32x2_t] + - [i8, int8x16x2_t, int8x16_t] + - [i16, int16x8x2_t, int16x8_t] + - [i32, int32x4x2_t, int32x4_t] + - [f32, float32x2x2_t, float32x2_t] + - [f32, float32x4x2_t, float32x4_t] + compose: + - LLVMLink: + name: 'st2.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst2]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st2]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u8, uint8x8x2_t, int8x8x2_t] + - [u16, uint16x4x2_t, int16x4x2_t] + - [u32, uint32x2x2_t, int32x2x2_t] + - [u8, uint8x16x2_t, int8x16x2_t] + - [u16, uint16x8x2_t, int16x8x2_t] + - [u32, uint32x4x2_t, int32x4x2_t] + - [p8, poly8x8x2_t, int8x8x2_t] + - [p16, poly16x4x2_t, int16x4x2_t] + - [p8, poly8x16x2_t, int8x16x2_t] + - [p16, poly16x8x2_t, int16x8x2_t] + compose: + - FnCall: + - "vst2{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x2_t, '3', int8x8_t] + - [i16, int16x4x2_t, '2', int16x4_t] + - [i32, int32x2x2_t, '1', int32x2_t] + - [i16, int16x8x2_t, '3', int16x8_t] + - [i32, int32x4x2_t, '2', int32x4_t] + - [f32, float32x2x2_t, '1', float32x2_t] + - [f32, float32x4x2_t, '2', float32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst2.{neon_type[1].lane_nox}' + arguments: + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst2, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [u8, uint8x8x2_t, int8x8x2_t, '3'] + - [u16, uint16x4x2_t, int16x4x2_t, '2'] + - [u32, uint32x2x2_t, int32x2x2_t, '1'] + - [u16, uint16x8x2_t, int16x8x2_t, '3'] + - [u32, uint32x4x2_t, int32x4x2_t, '2'] + - [p8, poly8x8x2_t, int8x8x2_t, '3'] + - [p16, poly16x4x2_t, int16x4x2_t, '2'] + - [p16, poly16x8x2_t, int16x8x2_t, '3'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vst2{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst2{neon_type[1].nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vst2] + safety: + unsafe: [neon] + types: + - [i8, int8x8x2_t, int8x8_t, '1'] + - [i16, int16x4x2_t, int16x4_t, '2'] + - [i32, int32x2x2_t, int32x2_t, '4'] + - [i8, int8x16x2_t, int8x16_t, '1'] + - [i16, int16x8x2_t, int16x8_t, '2'] + - [i32, int32x4x2_t, int32x4_t, '4'] + - [f32, float32x2x2_t, float32x2_t, '4'] + - [f32, float32x4x2_t, float32x4_t, '4'] + compose: + - LLVMLink: + name: 'vst2.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst2.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst2{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', "{type[3]}"]] + + - name: "vst2{neon_type[1].lane_nox}" + doc: "Store multiple 2-element structures from two registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst2, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x2_t, '3', int8x8_t, '1'] + - [i16, int16x4x2_t, '2', int16x4_t, '2'] + - [i32, int32x2x2_t, '1', int32x2_t, '4'] + - [i16, int16x8x2_t, '3', int16x8_t, '2'] + - [i32, int32x4x2_t, '2', int32x4_t, '4'] + - [f32, float32x4x2_t, '2', float32x4_t, '4'] + - [f32, float32x2x2_t, '1', float32x2_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst2lane.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst2lane.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst2{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'LANE', "{type[4]}"]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x3_t, int64x1_t] + compose: + - LLVMLink: + name: 'st3.{neon_type[1].nox}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v8 + - *neon-aes + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [p64, poly64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - "vst3{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x3_t, int64x1_t] + compose: + - LLVMLink: + name: 'vst3.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst3.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst3{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', '8']] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u64, uint64x1x3_t, int64x1x3_t] + compose: + - FnCall: + - "vst3{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst3, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [u8, uint8x8x3_t, int8x8x3_t, '3'] + - [u16, uint16x4x3_t, int16x4x3_t, '2'] + - [u32, uint32x2x3_t, int32x2x3_t, '1'] + - [u16, uint16x8x3_t, int16x8x3_t, '3'] + - [u32, uint32x4x3_t, int32x4x3_t, '2'] + - [p8, poly8x8x3_t, int8x8x3_t, '3'] + - [p16, poly16x4x3_t, int16x4x3_t, '2'] + - [p16, poly16x8x3_t, int16x8x3_t, '3'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vst3{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst3]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st3]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u8, uint8x8x3_t, int8x8x3_t] + - [u16, uint16x4x3_t, int16x4x3_t] + - [u32, uint32x2x3_t, int32x2x3_t] + - [u8, uint8x16x3_t, int8x16x3_t] + - [u16, uint16x8x3_t, int16x8x3_t] + - [u32, uint32x4x3_t, int32x4x3_t] + - [p8, poly8x8x3_t, int8x8x3_t] + - [p16, poly16x4x3_t, int16x4x3_t] + - [p8, poly8x16x3_t, int8x16x3_t] + - [p16, poly16x8x3_t, int16x8x3_t] + compose: + - FnCall: + - "vst3{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vst3] + safety: + unsafe: [neon] + types: + - [i8, int8x8x3_t, int8x8_t, '1'] + - [i16, int16x4x3_t, int16x4_t, '2'] + - [i32, int32x2x3_t, int32x2_t, '4'] + - [i8, int8x16x3_t, int8x16_t, '1'] + - [i16, int16x8x3_t, int16x8_t, '2'] + - [i32, int32x4x3_t, int32x4_t, '4'] + - [f32, float32x2x3_t, float32x2_t, '4'] + - [f32, float32x4x3_t, float32x4_t, '4'] + compose: + - LLVMLink: + name: 'vst3.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst3.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst3{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', "{type[3]}"]] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst3, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x3_t, '3', int8x8_t, '1'] + - [i16, int16x4x3_t, '2', int16x4_t, '2'] + - [i32, int32x2x3_t, '1', int32x2_t, '4'] + - [i16, int16x8x3_t, '3', int16x8_t, '2'] + - [i32, int32x4x3_t, '2', int32x4_t, '4'] + - [f32, float32x2x3_t, '1', float32x2_t, '4'] + - [f32, float32x4x3_t, '2', float32x4_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst3lane.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'c: {type[3]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst3lane.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', "{type[4]}"]] + + - name: "vst3{neon_type[1].nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*target-not-arm, *neon-stable] + assert_instr: [st3] + safety: + unsafe: [neon] + types: + - [i8, int8x8x3_t, int8x8_t] + - [i16, int16x4x3_t, int16x4_t] + - [i32, int32x2x3_t, int32x2_t] + - [i8, int8x16x3_t, int8x16_t] + - [i16, int16x8x3_t, int16x8_t] + - [i32, int32x4x3_t, int32x4_t] + - [f32, float32x2x3_t, float32x2_t] + - [f32, float32x4x3_t, float32x4_t] + compose: + - LLVMLink: + name: 'vst3.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] + + - name: "vst3{neon_type[1].lane_nox}" + doc: "Store multiple 3-element structures from three registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x3_t, '3', int8x8_t] + - [i16, int16x4x3_t, '2', int16x4_t] + - [i32, int32x2x3_t, '1', int32x2_t] + - [i16, int16x8x3_t, '3', int16x8_t] + - [i32, int32x4x3_t, '2', int32x4_t] + - [f32, float32x2x3_t, '1', float32x2_t] + - [f32, float32x4x3_t, '2', float32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst3.{neon_type[1].lane_nox}' + arguments: + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'c: {type[3]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v8 + - *neon-aes + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [p64, poly64x1x4_t, int64x1x4_t] + compose: + - FnCall: + - "vst4{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x4_t, int64x1_t] + compose: + - LLVMLink: + name: 'vst4.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst4.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst4{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', '8']] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - *neon-stable + assert_instr: [nop] + safety: + unsafe: [neon] + types: + - [i64, int64x1x4_t, int64x1_t] + compose: + - LLVMLink: + name: 'vst4.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4.{neon_type[2]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u64, uint64x1x4_t, int64x1x3_t] + compose: + - FnCall: + - "vst4{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst4, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [u8, uint8x8x4_t, int8x8x4_t, '3'] + - [u16, uint16x4x4_t, int16x4x4_t, '2'] + - [u32, uint32x2x4_t, int32x2x4_t, '1'] + - [u16, uint16x8x4_t, int16x8x4_t, '3'] + - [u32, uint32x4x4_t, int32x4x4_t, '2'] + - [p8, poly8x8x4_t, int8x8x4_t, '3'] + - [p16, poly16x4x4_t, int16x4x4_t, '2'] + - [p16, poly16x8x4_t, int16x8x4_t, '3'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vst4{neon_type[2].lane_nox}::" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst4]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st4]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [u8, uint8x8x4_t, int8x8x4_t] + - [u16, uint16x4x4_t, int16x4x4_t] + - [u32, uint32x2x4_t, int32x2x4_t] + - [u8, uint8x16x4_t, int8x16x4_t] + - [u16, uint16x8x4_t, int16x8x4_t] + - [u32, uint32x4x4_t, int32x4x4_t] + - [p8, poly8x8x4_t, int8x8x4_t] + - [p16, poly16x4x4_t, int16x4x4_t] + - [p8, poly8x16x4_t, int8x16x4_t] + - [p16, poly16x8x4_t, int16x8x4_t] + compose: + - FnCall: + - "vst4{neon_type[2].nox}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - *neon-unstable + assert_instr: [vst4] + safety: + unsafe: [neon] + types: + - [i8, int8x8x4_t, int8x8_t, '1'] + - [i16, int16x4x4_t, int16x4_t, '2'] + - [i32, int32x2x4_t, int32x2_t, '4'] + - [i8, int8x16x4_t, int8x16_t, '1'] + - [i16, int16x8x4_t, int16x8_t, '2'] + - [i32, int32x4x4_t, int32x4_t, '4'] + - [f32, float32x2x4_t, float32x2_t, '4'] + - [f32, float32x4x4_t, float32x4_t, '4'] + compose: + - LLVMLink: + name: 'vst4.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst4.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst4{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', "{type[3]}"]] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst4, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-unstable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x4_t, '3', int8x8_t, '1'] + - [i16, int16x4x4_t, '2', int16x4_t, '2'] + - [i32, int32x2x4_t, '1', int32x2_t, '4'] + - [i16, int16x8x4_t, '3', int16x8_t, '2'] + - [i32, int32x4x4_t, '2', int32x4_t, '4'] + - [f32, float32x2x4_t, '1', float32x2_t, '4'] + - [f32, float32x4x4_t, '2', float32x4_t, '4'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst4lane.{neon_type[1]}' + arguments: + - 'ptr: *mut i8' + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'c: {type[3]}' + - 'd: {type[3]}' + - 'n: i32' + - 'size: i32' + links: + - link: 'llvm.arm.neon.vst4lane.p0i8.v{neon_type[1].lane}{type[0]}' + arch: arm + - FnCall: ['_vst4{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', 'LANE', "{type[4]}"]] + + - name: "vst4{neon_type[1].nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: [*target-not-arm, *neon-stable] + assert_instr: [st4] + safety: + unsafe: [neon] + types: + - [i8, int8x8x4_t, int8x8_t] + - [i16, int16x4x4_t, int16x4_t] + - [i32, int32x2x4_t, int32x2_t] + - [i8, int8x16x4_t, int8x16_t] + - [i16, int16x8x4_t, int16x8_t] + - [i32, int32x4x4_t, int32x4_t] + - [f32, float32x2x4_t, float32x2_t] + - [f32, float32x4x4_t, float32x4_t] + compose: + - LLVMLink: + name: 'vst4.{neon_type[1]}' + arguments: + - 'a: {type[2]}' + - 'b: {type[2]}' + - 'c: {type[2]}' + - 'd: {type[2]}' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] + + - name: "vst4{neon_type[1].lane_nox}" + doc: "Store multiple 4-element structures from four registers" + arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] + attr: + - *target-not-arm + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] + - *neon-stable + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [i8, int8x8x4_t, '3', int8x8_t] + - [i16, int16x4x4_t, '2', int16x4_t] + - [i32, int32x2x4_t, '1', int32x2_t] + - [i16, int16x8x4_t, '3', int16x8_t] + - [i32, int32x4x4_t, '2', int32x4_t] + - [f32, float32x2x4_t, '1', float32x2_t] + - [f32, float32x4x4_t, '2', float32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - LLVMLink: + name: 'vst4.{neon_type[1].lane_nox}' + arguments: + - 'a: {type[3]}' + - 'b: {type[3]}' + - 'c: {type[3]}' + - 'd: {type[3]}' + - 'n: i64' + - 'ptr: *mut i8' + links: + - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' + arch: aarch64,arm64ec + - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] + + - name: "vusdot{neon_type[0].no}" + doc: "Dot product vector form with unsigned and signed integers" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-i8mm + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vusdot]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usdot]]}]] + - *neon-unstable-i8mm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int32x2_t, uint8x8_t, int8x8_t] + - [int32x4_t, uint8x16_t, int8x16_t] + compose: + - LLVMLink: + name: "usdot.{neon_type[0]}" + links: + - link: "llvm.aarch64.neon.usdot.v{neon_type[0].lane}i32.v{neon_type[1].lane}i8" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.usdot.v{neon_type[0].lane}i32.v{neon_type[1].lane}i8" + arch: arm + + - name: "vusdot{type[0]}" + doc: "Dot product index form with unsigned and signed integers" + arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}", "c: int8x8_t"] + return_type: "{neon_type[1]}" + attr: + - *neon-i8mm + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vusdot, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usdot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-i8mm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - ['_lane_s32', int32x2_t, uint8x8_t, '[LANE as u32, LANE as u32]'] + - ['q_lane_s32', int32x4_t, uint8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Let: + - c + - int32x2_t + - FnCall: [transmute, [c]] + - Let: + - c + - "{type[1]}" + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + - FnCall: ["vusdot{neon_type[1].no}", [a, b, {FnCall: [transmute, [c]]}]] + + - name: "vsudot{neon_type[0].lane_nox}" + doc: "Dot product index form with signed and unsigned integers" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-i8mm + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsudot, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sudot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - *neon-unstable-i8mm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [int32x2_t, int8x8_t, uint8x8_t, '[LANE as u32, LANE as u32]', uint32x2_t] + - [int32x4_t, int8x16_t, uint8x8_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]', uint32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Let: + - c + - uint32x2_t + - FnCall: [transmute, [c]] + - Let: + - c + - "{type[4]}" + - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] + - FnCall: ["vusdot{neon_type[0].no}", [a, {FnCall: [transmute, [c]]}, b]] + + - name: "vmul{neon_type[1].no}" + doc: Multiply + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmul{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['.i8', int8x8_t] + - ['.i8', int8x16_t] + - ['.i16', int16x4_t] + - ['.i16', int16x8_t] + - ['.i32', int32x2_t] + - ['.i32', int32x4_t] + - ['.i8', uint8x8_t] + - ['.i8', uint8x16_t] + - ['.i16', uint16x4_t] + - ['.i16', uint16x8_t] + - ['.i32', uint32x2_t] + - ['.i32', uint32x4_t] + compose: + - FnCall: [simd_mul, [a, b]] + + - name: "vmul{neon_type[1].no}" + doc: Multiply + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmul.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmul]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [f32, float32x2_t] + - [f32, float32x4_t] + compose: + - FnCall: [simd_mul, [a, b]] + + - name: "vmul{neon_type[0].lane_nox}" + doc: Multiply + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmul, 'LANE = 1']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] + - FnCall: + - simd_mul + - - a + - FnCall: ["simd_shuffle!", [b, b, "{type[3]}"]] + + - name: "vmul{neon_type[0].laneq_nox}" + doc: Multiply + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmul, 'LANE = 1']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - *neon-stable-not-arm + - *neon-unstable-is-arm + static_defs: ["const LANE: i32"] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] + - FnCall: + - simd_mul + - - a + - FnCall: ["simd_shuffle!", [b, b, "{type[3]}"]] + + - name: "vmull{neon_type[1].no}" + doc: Signed multiply long + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smull]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["s8", int8x8_t, int16x8_t] + - ["s16", int16x4_t, int32x4_t] + - ["s32", int32x2_t, int64x2_t] + compose: + - LLVMLink: + name: "smull.{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.smull.{neon_type[1]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vmulls.{neon_type[1]}" + arch: arm + + - name: "vmull{neon_type[1].no}" + doc: "Unsigned multiply long" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umull]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["u8", uint8x8_t, uint16x8_t] + - ["u16", uint16x4_t, uint32x4_t] + - ["u32", uint32x2_t, uint64x2_t] + compose: + - LLVMLink: + name: "smull.{neon_type[1]}" + links: + - link: "llvm.aarch64.neon.umull.{neon_type[1]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vmullu.{neon_type[1]}" + arch: arm + + - name: "vmull{neon_type[1].no}" + doc: "Polynomial multiply long" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [pmull]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ["p8", poly8x8_t, poly16x8_t, int8x8_t] + compose: + - LLVMLink: + name: "pmull.{neon_type[1].no}" + links: + - link: "llvm.aarch64.neon.pmull.{neon_type[3]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vmullp.{neon_type[3]}" + arch: arm + + - name: "vmull_n{neon_type[0].no}" + doc: Vector long multiply with scalar + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vmull"]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smull]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", int32x4_t] + - [int32x2_t, "i32", int64x2_t] + compose: + - FnCall: + - "vmull{neon_type[0].no}" + - - a + - FnCall: + - "vdup_n{neon_type[0].no}" + - - b + + - name: "vmull_n{neon_type[0].no}" + doc: Vector long multiply with scalar + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vmull"]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umull]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint16x4_t, "u16", uint32x4_t] + - [uint32x2_t, "u32", uint64x2_t] + compose: + - FnCall: + - "vmull{neon_type[0].no}" + - - a + - FnCall: + - "vdup_n{neon_type[0].no}" + - - b + + - name: "vfma{neon_type.no}" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - vfma + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - fmla + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "fma.{neon_type}" + links: + - link: "llvm.fma.{neon_type}" + arch: aarch64 + - link: "llvm.fma.{neon_type}" + arch: arm + - FnCall: ["_vfma{neon_type.no}", [b, c, a]] + + - name: "vfma{neon_type[0].N}" + doc: Floating-point fused Multiply-Add to accumulator(vector) + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - vfma + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - fmla + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - [float32x2_t, f32] + - [float32x4_t, f32] + compose: + - FnCall: + - "vfma{neon_type[0].no}" + - - a + - b + - FnCall: + - "vdup{neon_type[0].N}_vfp4" + - - c + + - name: "vsub{neon_type[1].no}" + doc: "Subtract" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vsub{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['.i8', int8x8_t] + - ['.i8', int8x16_t] + - ['.i16', int16x4_t] + - ['.i16', int16x8_t] + - ['.i32', int32x2_t] + - ['.i32', int32x4_t] + - ['.i8', uint8x8_t] + - ['.i8', uint8x16_t] + - ['.i16', uint16x4_t] + - ['.i16', uint16x8_t] + - ['.i32', uint32x2_t] + - ['.i32', uint32x4_t] + - ['.i64', int64x1_t] + - ['.i64', int64x2_t] + - ['.i64', uint64x1_t] + - ['.i64', uint64x2_t] + compose: + - FnCall: [simd_sub, [a, b]] + + - name: "vsub{neon_type[1].no}" + doc: "Subtract" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vsub.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fsub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['f32', float32x2_t] + - ['f32', float32x4_t] + compose: + - FnCall: [simd_sub, [a, b]] + + - name: "vadd{neon_type.no}" + doc: Bitwise exclusive OR + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - target_feature + - - 'enable = "v7"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - nop + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - nop + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - poly8x8_t + - poly16x4_t + - poly8x16_t + - poly16x8_t + - poly64x1_t + - poly64x2_t + compose: + - FnCall: + - simd_xor + - - a + - b + + - name: "vaddq_{type}" + doc: Bitwise exclusive OR + arguments: ["a: {type}", "b: {type}"] + return_type: "{type}" + attr: + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - target_feature + - - 'enable = "v7"' + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - 'target_arch = "arm"' + - FnCall: + - assert_instr + - - nop + - FnCall: + - cfg_attr + - - FnCall: + - all + - - test + - FnCall: + - any + - - 'target_arch = "aarch64"' + - 'target_arch = "arm64ec"' + - FnCall: + - assert_instr + - - nop + - FnCall: + - cfg_attr + - - FnCall: + - not + - - 'target_arch = "arm"' + - FnCall: + - stable + - - 'feature = "neon_intrinsics"' + - 'since = "1.59.0"' + - FnCall: + - cfg_attr + - - 'target_arch = "arm"' + - FnCall: + - unstable + - - 'feature = "stdarch_arm_neon_intrinsics"' + - 'issue = "111800"' + safety: + unsafe: [neon] + types: + - p128 + compose: + - Xor: + - a + - b + + - name: "vsubhn{neon_type[0].noq}" + doc: Subtract returning high narrow + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vsubhn"]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [subhn]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'i16x8', 'i16x8::new(8, 8, 8, 8, 8, 8, 8, 8)'] + - [int32x4_t, int16x4_t, 'i32x4', 'i32x4::new(16, 16, 16, 16)'] + - [int64x2_t, int32x2_t, 'i64x2', 'i64x2::new(32, 32)'] + - [uint16x8_t, uint8x8_t, 'u16x8', 'u16x8::new(8, 8, 8, 8, 8, 8, 8, 8)'] + - [uint32x4_t, uint16x4_t, 'u32x4', 'u32x4::new(16, 16, 16, 16)'] + - [uint64x2_t, uint32x2_t, 'u64x2', 'u64x2::new(32, 32)'] + compose: + - Let: [c, "{type[2]}", "{type[3]}"] + - FnCall: + - simd_cast + - - FnCall: + - simd_shr + - - FnCall: [simd_sub, [a, b]] + - FnCall: [transmute, [c]] + + - name: "vsubhn_high{neon_type[1].noq}" + doc: Subtract returning high narrow + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vsubhn"]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [subhn2]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] + - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] + - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] + - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] + compose: + - Let: + - d + - "{neon_type[0]}" + - FnCall: ["vsubhn{neon_type[1].noq}", [b, c]] + - FnCall: [simd_shuffle!, [a, d, "{type[3]}"]] + + - name: "vhsub{neon_type[1].no}" + doc: "Signed halving subtract" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vhsub.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uhsub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['u8', uint8x8_t] + - ['u8', uint8x16_t] + - ['u16', uint16x4_t] + - ['u16', uint16x8_t] + - ['u32', uint32x2_t] + - ['u32', uint32x4_t] + compose: + - LLVMLink: + name: "uhsub.{neon_type[1].no}" + links: + - link: "llvm.aarch64.neon.uhsub.{neon_type[1]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vhsubu.{neon_type[1]}" + arch: arm + + - name: "vhsub{neon_type[1].no}" + doc: "Signed halving subtract" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vhsub.{type[0]}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [shsub]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - ['s8', int8x8_t] + - ['s8', int8x16_t] + - ['s16', int16x4_t] + - ['s16', int16x8_t] + - ['s32', int32x2_t] + - ['s32', int32x4_t] + compose: + - LLVMLink: + name: "shsub.{neon_type[1].no}" + links: + - link: "llvm.aarch64.neon.shsub.{neon_type[1]}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vhsubs.{neon_type[1]}" + arch: arm + + - name: "vsubw{neon_type[1].noq}" + doc: Signed Subtract Wide + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubw]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ssubw]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t] + - [int32x4_t, int16x4_t] + - [int64x2_t, int32x2_t] + compose: + - FnCall: + - simd_sub + - - a + - FnCall: [simd_cast, [b]] + + - name: "vsubw{neon_type[1].noq}" + doc: Unsigned Subtract Wide + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubw]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usubw]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t] + - [uint32x4_t, uint16x4_t] + - [uint64x2_t, uint32x2_t] + compose: + - FnCall: + - simd_sub + - - a + - FnCall: [simd_cast, [b]] + + - name: "vsubl{neon_type[0].noq}" + doc: "Signed Subtract Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubl]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ssubl]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t] + - [int16x4_t, int32x4_t] + - [int32x2_t, int64x2_t] + compose: + - Let: + - c + - "{neon_type[1]}" + - FnCall: [simd_cast, [a]] + - Let: + - d + - "{neon_type[1]}" + - FnCall: [simd_cast, [b]] + - FnCall: [simd_sub, [c, d]] + + - name: "vsubl{neon_type[0].noq}" + doc: "Unsigned Subtract Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubl]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usubl]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint16x8_t] + - [uint16x4_t, uint32x4_t] + - [uint32x2_t, uint64x2_t] + compose: + - Let: + - c + - "{neon_type[1]}" + - FnCall: [simd_cast, [a]] + - Let: + - d + - "{neon_type[1]}" + - FnCall: [simd_cast, [b]] + - FnCall: [simd_sub, [c, d]] + + - name: "vdot{neon_type[0].no}" + doc: Dot product arithmetic (vector) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v8 + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsdot]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sdot]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int32x2_t, int8x8_t] + - [int32x4_t, int8x16_t] + compose: + - LLVMLink: + name: "sdot.{neon_type[0]}.{neon_type[1]}" + links: + - link: "llvm.arm.neon.sdot.{neon_type[0]}.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.sdot.{neon_type[0]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vdot{neon_type[0].no}" + doc: Dot product arithmetic (vector) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - *neon-v8 + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vudot]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [udot]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint32x2_t, uint8x8_t] + - [uint32x4_t, uint8x16_t] + compose: + - LLVMLink: + name: "udot.{neon_type[0]}.{neon_type[1]}" + links: + - link: "llvm.arm.neon.udot.{neon_type[0]}.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.udot.{neon_type[0]}.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vdot{neon_type[0].lane_nox}" + doc: Dot product arithmetic (indexed) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + static_defs: ["const LANE: i32"] + attr: + - *neon-v8 + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsdot, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sdot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int32x2_t, int8x8_t, int8x8_t, int32x2_t, '[LANE as u32, LANE as u32]'] + - [int32x4_t, int8x16_t, int8x8_t, int32x2_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Let: + - c + - "{neon_type[3]}" + - FnCall: [transmute, [c]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] + - FnCall: + - "vdot{neon_type[0].no}" + - - a + - b + - FnCall: [transmute, [c]] + + - name: "vdot{neon_type[0].lane_nox}" + doc: Dot product arithmetic (indexed) + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[0]}" + static_defs: ["const LANE: i32"] + attr: + - *neon-v8 + - FnCall: [target_feature, ['enable = "neon,dotprod"']] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vudot, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [udot, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint32x2_t, uint8x8_t, uint8x8_t, uint32x2_t, '[LANE as u32, LANE as u32]'] + - [uint32x4_t, uint8x16_t, uint8x8_t, uint32x2_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - Let: + - c + - "{neon_type[3]}" + - FnCall: [transmute, [c]] + - Let: + - c + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] + - FnCall: + - "vdot{neon_type[0].no}" + - - a + - b + - FnCall: [transmute, [c]] + + - name: "vmax{neon_type.no}" + doc: Maximum (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smax]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "smax.{neon_type}" + links: + - link: "llvm.arm.neon.vmaxs.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.smax.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmax{neon_type.no}" + doc: Maximum (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umax]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "smax.{neon_type}" + links: + - link: "llvm.arm.neon.vmaxu.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.umax.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmax{neon_type.no}" + doc: Maximum (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmax]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "smax.{neon_type}" + links: + - link: "llvm.arm.neon.vmaxs.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.fmax.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmaxnm{neon_type.no}" + doc: Floating-point Maximum Number (vector) + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmaxnm]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmaxnm]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "fmaxnm.{neon_type}" + links: + - link: "llvm.arm.neon.vmaxnm.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.fmaxnm.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmin{neon_type.no}" + doc: "Minimum (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smin]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "smin.{neon_type}" + links: + - link: "llvm.arm.neon.vmins.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.smin.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmin{neon_type.no}" + doc: "Minimum (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umin]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "umin.{neon_type}" + links: + - link: "llvm.arm.neon.vminu.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.umin.{neon_type}" + arch: aarch64,arm64ec + + - name: "vmin{neon_type.no}" + doc: "Minimum (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmin]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "fmin.{neon_type}" + links: + - link: "llvm.arm.neon.vmins.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.fmin.{neon_type}" + arch: aarch64,arm64ec + + - name: "vminnm{neon_type.no}" + doc: "Floating-point Minimum Number (vector)" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vminnm]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fminnm]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "fminnm.{neon_type}" + links: + - link: "llvm.arm.neon.vminnm.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.fminnm.{neon_type}" + arch: aarch64,arm64ec + + - name: "vpadd{neon_type.no}" + doc: Floating-point add pairwise + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vpadd]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [faddp]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - float32x2_t + compose: + - LLVMLink: + name: "faddp.{neon_type}" + links: + - link: "llvm.arm.neon.vpadd.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.faddp.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqdmull{neon_type[0].noq}" + doc: "Signed saturating doubling multiply long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, int32x4_t] + - [int32x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vqdmull{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vqdmull.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.sqdmull.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqdmull_n{neon_type[0].no}" + doc: "Vector saturating doubling long multiply with scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", int32x4_t] + - [int32x2_t, "i32", int64x2_t] + compose: + - FnCall: ["vqdmull{neon_type[0].noq}", [a, {FnCall: ["vdup_n{neon_type[0].noq}", [b]]}]] + + - name: "vqdmull_lane_s16" + doc: "Vector saturating doubling long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, int32x4_t, '[N as u32, N as u32, N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - FnCall: [vqdmull_s16, [a, b]] + + - name: "vqdmull_lane_s32" + doc: "Vector saturating doubling long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull, 'N = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, int32x2_t, int64x2_t, '[N as u32, N as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '1']] + - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] + - FnCall: [vqdmull_s32, [a, b]] + + - name: "vqdmlal{neon_type[1].noq}" + doc: "Signed saturating doubling multiply-add long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] + - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] + compose: + - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{neon_type[2].noq}", [b, c]]}]] + + - name: "vqdmlal_n{neon_type[1].noq}" + doc: "Vector widening saturating doubling multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, "i16", int32x4_t] + - [int64x2_t, int32x2_t, "i32", int64x2_t] + compose: + - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull_n{neon_type[1].noq}", [b, c]]}]] + + - name: "vqdmlal_lane_s16" + doc: "Vector widening saturating doubling multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal, N = 2]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - FnCall: [vqaddq_s32, [a, {FnCall: ["vqdmull_lane_s16::", [b, c]]}]] + + - name: "vqdmlal_lane_s32" + doc: "Vector widening saturating doubling multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal, N = 1]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '1']] + - FnCall: [vqaddq_s64, [a, {FnCall: ["vqdmull_lane_s32::", [b, c]]}]] + + - name: "vqdmlsl{neon_type[1].noq}" + doc: "Signed saturating doubling multiply-subtract long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] + - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] + compose: + - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{neon_type[1].noq}", [b, c]]}]] + + - name: "vqdmlsl{type[4]}" + doc: "Vector widening saturating doubling multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, "i16", int32x4_t, '_n_s16'] + - [int64x2_t, int32x2_t, "i32", int64x2_t, '_n_s32'] + compose: + - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] + + - name: "vqdmlsl_lane_s16" + doc: "Vector widening saturating doubling multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl, N = 2]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '2']] + - FnCall: [vqsubq_s32, [a, {FnCall: ["vqdmull_lane_s16::", [b, c]]}]] + + - name: "vqdmlsl_lane_s32" + doc: "Vector widening saturating doubling multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl, N = 1]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl, 'N = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, '1']] + - FnCall: [vqsubq_s64, [a, {FnCall: ["vqdmull_lane_s32::", [b, c]]}]] + + - name: "vqdmulh{neon_type[0].no}" + doc: "Signed saturating doubling multiply returning high half" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, int16x4_t] + - [int16x8_t, int16x8_t, int16x8_t] + - [int32x2_t, int32x2_t, int32x2_t] + - [int32x4_t, int32x4_t, int32x4_t] + compose: + - LLVMLink: + name: "vqdmulh{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vqdmulh.{neon_type[0]}" + arch: arm + - link: "llvm.aarch64.neon.sqdmulh.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vqdmulh{type[3]}" + doc: "Vector saturating doubling multiply high with scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", int16x4_t, '_n_s16'] + - [int32x2_t, "i32", int32x2_t, '_n_s32'] + - [int16x8_t, "i16", int16x8_t, 'q_n_s16'] + - [int32x4_t, "i32", int32x4_t, 'q_n_s32'] + compose: + - Let: [b, "{neon_type[0]}", {FnCall: ["vdup{type[3]}", [b]]}] + - FnCall: ["vqdmulh{neon_type[0].no}", [a, b]] + + - name: "vqmovn{neon_type[0].noq}" + doc: "Signed saturating extract narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqxtn]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t] + - [int32x4_t, int16x4_t] + - [int64x2_t, int32x2_t] + compose: + - LLVMLink: + name: "vqmovn{neon_type[0].noq}" + links: + - link: "llvm.arm.neon.vqmovns.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.sqxtn.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqmovun{neon_type[0].noq}" + doc: "Signed saturating extract unsigned narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovun]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqxtun]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, uint8x8_t] + - [int32x4_t, uint16x4_t] + - [int64x2_t, uint32x2_t] + compose: + - LLVMLink: + name: "vqmovun{neon_type[0].noq}" + links: + - link: "llvm.arm.neon.vqmovnsu.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.sqxtun.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqrdmulh{neon_type[0].no}" + doc: "Signed saturating rounding doubling multiply returning high half" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, int16x4_t] + - [int16x8_t, int16x8_t, int16x8_t] + - [int32x2_t, int32x2_t, int32x2_t] + - [int32x4_t, int32x4_t, int32x4_t] + compose: + - LLVMLink: + name: "vqrdmulh{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vqrdmulh.{neon_type[0]}" + arch: arm + - link: "llvm.aarch64.neon.sqrdmulh.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vqrshl{neon_type.no}" + doc: "Signed saturating rounding shift left" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "vqrshl{neon_type}" + links: + - link: "llvm.arm.neon.vqrshifts.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.sqrshl.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqrshl{neon_type[0].no}" + doc: "Unsigned signed saturating rounding shift left" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [uint16x4_t, int16x4_t] + - [uint16x8_t, int16x8_t] + - [uint32x2_t, int32x2_t] + - [uint32x4_t, int32x4_t] + - [uint64x1_t, int64x1_t] + - [uint64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vqrshl{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vqrshiftu.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.uqrshl.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqrshrn_n{neon_type[0].noq}" + doc: "Signed saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqrshrn{neon_type[0].noq}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqrshiftns.{neon_type[1]}" + arch: arm + - FnCall: ["_vqrshrn_n{neon_type[0].noq}", [a, '{type[3]}']] + + - name: "vqrshrn_n{neon_type[0].noq}" + doc: "Signed saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqrshrn{neon_type[0].no}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.sqrshrn.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqrshrn_n{neon_type[0].noq}", [a, N]] + + - name: "vqrshrun_n{neon_type[0].noq}" + doc: "Signed saturating rounded shift right unsigned narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] + - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] + - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqrshrun_n{neon_type[0].noq}" + arguments: + - 'a: {neon_type[0]}' + - 'n: {neon_type[0]}' + links: + - link: "llvm.arm.neon.vqrshiftnsu.{neon_type[1]}" + arch: arm + - FnCall: + - "_vqrshrun_n{neon_type[0].noq}" + - - a + - "{type[3]}" + + - name: "vqrshrun_n{neon_type[0].noq}" + doc: "Signed saturating rounded shift right unsigned narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] + - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] + - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqrshrun_n{neon_type[0].noq}" + arguments: + - 'a: {neon_type[0]}' + - 'n: i32' + links: + - link: "llvm.aarch64.neon.sqrshrun.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqrshrun_n{neon_type[0].noq}", [a, N]] + + - name: "vqshl{neon_type.no}" + doc: "Signed saturating shift left" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "vqshl{neon_type}" + links: + - link: "llvm.arm.neon.vqshifts.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.sqshl.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqshl{neon_type[0].N}" + doc: "Signed saturating shift left" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqshl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, '3'] + - [int8x16_t, '3'] + - [int16x4_t, '4'] + - [int16x8_t, '4'] + - [int32x2_t, '5'] + - [int32x4_t, '5'] + - [int64x1_t, '6'] + - [int64x2_t, '6'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] + - FnCall: + - "vqshl{neon_type[0].no}" + - - a + - FnCall: ["vdup{neon_type[0].N}", ['N as _']] + + - name: "vqshl{neon_type[0].no}" + doc: "Unsigned saturating shift left" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [uint16x4_t, int16x4_t] + - [uint16x8_t, int16x8_t] + - [uint32x2_t, int32x2_t] + - [uint32x4_t, int32x4_t] + - [uint64x1_t, int64x1_t] + - [uint64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vqshl{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vqshiftu.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.uqshl.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vqshl{neon_type[0].N}" + doc: "Unsigned saturating shift left" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl, N = 2]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqshl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, '3', int8x8_t] + - [uint8x16_t, '3', int8x16_t] + - [uint16x4_t, '4', int16x4_t] + - [uint16x8_t, '4', int16x8_t] + - [uint32x2_t, '5', int32x2_t] + - [uint32x4_t, '5', int32x4_t] + - [uint64x1_t, '6', int64x1_t] + - [uint64x2_t, '6', int64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] + - FnCall: + - "vqshl{neon_type[0].no}" + - - a + - FnCall: ["vdup{neon_type[2].N}", ['N as _']] + + - name: "vqshrn_n{neon_type[0].noq}" + doc: "Signed saturating shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrn{neon_type[0].no}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqshiftns.{neon_type[1]}" + arch: arm + - FnCall: ["_vqshrn_n{neon_type[0].noq}", [a, "{type[3]}"]] + + - name: "vqshrn_n{neon_type[0].noq}" + doc: "Signed saturating shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrn_n{neon_type[0].noq}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.sqshrn.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqshrn_n{neon_type[0].noq}", [a, N]] + + - name: "vqshrn_n_{neon_type[0]}" + doc: "Unsigned saturating shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { uint16x8_t([-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16]) }.as_signed()'] + - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }.as_signed()'] + - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { uint64x2_t([-N as u64, -N as u64]) }.as_signed()'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrn_n_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqshiftnu.{neon_type[1]}" + arch: arm + - FnCall: ["_vqshrn_n_{neon_type[0]}", ["a.as_signed()", "{type[3]}"]] + + - name: "vqshrn_n_{neon_type[0]}" + doc: "Unsigned saturating shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] + - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] + - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrn{neon_type[1].no}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.uqshrn.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqshrn_n_{neon_type[0]}", ["a.as_signed()", N]] + + - name: "vqshrun_n_{neon_type[0]}" + doc: "Signed saturating shift right unsigned narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] + - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] + - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrun_n_{neon_type[1]}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqshiftnsu.{neon_type[1]}" + arch: arm + - FnCall: ["_vqshrun_n_{neon_type[0]}", [a, "{type[3]}"]] + + - name: "vqshrun_n_{neon_type[0]}" + doc: "Signed saturating shift right unsigned narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] + - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] + - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vqshrun_n_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.sqshrun.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqshrun_n_{neon_type[0]}", [a, N]] + + - name: "vrsqrts{neon_type.no}" + doc: "Floating-point reciprocal square root step" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrts]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frsqrts]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vrsqrts{neon_type.no}" + links: + - link: "llvm.arm.neon.vrsqrts.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.frsqrts.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrecpe{neon_type.no}" + doc: "Reciprocal estimate." + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecpe]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frecpe]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vrecpe{neon_type.no}" + links: + - link: "llvm.arm.neon.vrecpe.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.frecpe.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrecps{neon_type.no}" + doc: "Floating-point reciprocal step" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecps]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frecps]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vrecps{neon_type.no}" + links: + - link: "llvm.arm.neon.vrecps.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.frecps.{neon_type}" + arch: aarch64,arm64ec + + - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" + doc: Vector reinterpret cast operation + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-aes + - *neon-v8 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [poly64x1_t, int32x2_t] + - [poly64x1_t, uint32x2_t] + - [poly64x2_t, int32x4_t] + - [poly64x2_t, uint32x4_t] + - [p128, int64x2_t] + - [p128, uint64x2_t] + - [p128, poly64x2_t] + - [poly8x16_t, p128] + - [p128, int8x16_t] + - [p128, uint8x16_t] + - [p128, poly8x16_t] + - [int32x2_t, poly64x1_t] + - [uint32x2_t, poly64x1_t] + - [int32x4_t, poly64x2_t] + - [uint32x4_t, poly64x2_t] + - [int64x2_t, p128] + - [uint64x2_t, p128] + - [poly64x2_t, p128] + - [poly64x1_t, int16x4_t] + - [poly64x1_t, uint16x4_t] + - [poly64x1_t, poly16x4_t] + - [poly64x2_t, int16x8_t] + - [poly64x2_t, uint16x8_t] + - [poly64x2_t, poly16x8_t] + - [p128, int32x4_t] + - [p128, uint32x4_t] + - [poly16x4_t, poly64x1_t] + - [int16x4_t, poly64x1_t] + - [uint16x4_t, poly64x1_t] + - [poly16x8_t, poly64x2_t] + - [int16x8_t, poly64x2_t] + - [uint16x8_t, poly64x2_t] + - [int32x4_t, p128] + - [uint32x4_t, p128] + - [poly64x1_t, int8x8_t] + - [poly64x1_t, uint8x8_t] + - [poly64x1_t, poly8x8_t] + - [poly64x2_t, int8x16_t] + - [poly64x2_t, uint8x16_t] + - [poly64x2_t, poly8x16_t] + - [p128, int16x8_t] + - [p128, uint16x8_t] + - [p128, poly16x8_t] + - [poly8x8_t, poly64x1_t] + - [int8x8_t, poly64x1_t] + - [uint8x8_t, poly64x1_t] + - [poly8x16_t, poly64x2_t] + - [int8x16_t, poly64x2_t] + - [uint8x16_t, poly64x2_t] + - [int16x8_t, p128] + - [uint16x8_t, p128] + - [poly16x8_t, p128] + - [int8x16_t, p128] + - [uint8x16_t, p128] + compose: + - FnCall: [transmute, [a]] + + - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" + doc: Vector reinterpret cast operation + arguments: ["a: {type[0]}"] + return_type: "{type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [poly8x8_t, int8x8_t] + - [poly16x4_t, int16x4_t] + - [uint16x4_t, int16x4_t] + - [uint32x2_t, int32x2_t] + - [uint64x1_t, int64x1_t] + - [uint8x16_t, int8x16_t] + - [poly8x16_t, int8x16_t] + - [poly16x8_t, int16x8_t] + - [uint16x8_t, int16x8_t] + - [uint32x4_t, int32x4_t] + - [uint64x2_t, int64x2_t] + - [poly8x8_t, uint8x8_t] + - [int8x8_t, uint8x8_t] + - [poly16x4_t, uint16x4_t] + - [int16x4_t, uint16x4_t] + - [int32x2_t, uint32x2_t] + - [int64x1_t, uint64x1_t] + - [poly8x16_t, uint8x16_t] + - [int8x16_t, uint8x16_t] + - [poly16x8_t, uint16x8_t] + - [int16x8_t, uint16x8_t] + - [int32x4_t, uint32x4_t] + - [int64x2_t, uint64x2_t] + - [int8x8_t, poly8x8_t] + - [uint8x8_t, poly8x8_t] + - [int16x4_t, poly16x4_t] + - [uint16x4_t, poly16x4_t] + - [int8x16_t, poly8x16_t] + - [uint8x16_t, poly8x16_t] + - [int16x8_t, poly16x8_t] + - [uint16x8_t, poly16x8_t] + - [int16x4_t, int8x8_t] + - [uint16x4_t, int8x8_t] + - [poly16x4_t, int8x8_t] + - [int32x2_t, int16x4_t] + - [uint32x2_t, int16x4_t] + - [int64x1_t, int32x2_t] + - [uint64x1_t, int32x2_t] + - [int16x8_t, int8x16_t] + - [uint16x8_t, int8x16_t] + - [poly16x8_t, int8x16_t] + - [int32x4_t, int16x8_t] + - [uint32x4_t, int16x8_t] + - [int64x2_t, int32x4_t] + - [uint64x2_t, int32x4_t] + - [poly16x4_t, uint8x8_t] + - [int16x4_t, uint8x8_t] + - [uint16x4_t, uint8x8_t] + - [int32x2_t, uint16x4_t] + - [uint32x2_t, uint16x4_t] + - [int64x1_t, uint32x2_t] + - [uint64x1_t, uint32x2_t] + - [poly16x8_t, uint8x16_t] + - [int16x8_t, uint8x16_t] + - [uint16x8_t, uint8x16_t] + - [int32x4_t, uint16x8_t] + - [uint32x4_t, uint16x8_t] + - [int64x2_t, uint32x4_t] + - [uint64x2_t, uint32x4_t] + - [poly16x4_t, poly8x8_t] + - [int16x4_t, poly8x8_t] + - [uint16x4_t, poly8x8_t] + - [int32x2_t, poly16x4_t] + - [uint32x2_t, poly16x4_t] + - [poly16x8_t, poly8x16_t] + - [int16x8_t, poly8x16_t] + - [uint16x8_t, poly8x16_t] + - [int32x4_t, poly16x8_t] + - [uint32x4_t, poly16x8_t] + - [poly8x8_t, int16x4_t] + - [int8x8_t, int16x4_t] + - [uint8x8_t, int16x4_t] + - [poly16x4_t, int32x2_t] + - [int16x4_t, int32x2_t] + - [uint16x4_t, int32x2_t] + - [int32x2_t, int64x1_t] + - [uint32x2_t, int64x1_t] + - [poly8x16_t, int16x8_t] + - [int8x16_t, int16x8_t] + - [uint8x16_t, int16x8_t] + - [poly16x8_t, int32x4_t] + - [int16x8_t, int32x4_t] + - [uint16x8_t, int32x4_t] + - [int32x4_t, int64x2_t] + - [uint32x4_t, int64x2_t] + - [poly8x8_t, uint16x4_t] + - [int8x8_t, uint16x4_t] + - [uint8x8_t, uint16x4_t] + - [poly16x4_t, uint32x2_t] + - [int16x4_t, uint32x2_t] + - [uint16x4_t, uint32x2_t] + - [int32x2_t, uint64x1_t] + - [uint32x2_t, uint64x1_t] + - [poly8x16_t, uint16x8_t] + - [int8x16_t, uint16x8_t] + - [uint8x16_t, uint16x8_t] + - [poly16x8_t, uint32x4_t] + - [int16x8_t, uint32x4_t] + - [uint16x8_t, uint32x4_t] + - [int32x4_t, uint64x2_t] + - [uint32x4_t, uint64x2_t] + - [poly8x8_t, poly16x4_t] + - [int8x8_t, poly16x4_t] + - [uint8x8_t, poly16x4_t] + - [poly8x16_t, poly16x8_t] + - [int8x16_t, poly16x8_t] + - [uint8x16_t, poly16x8_t] + - [int32x2_t, int8x8_t] + - [uint32x2_t, int8x8_t] + - [int64x1_t, int16x4_t] + - [uint64x1_t, int16x4_t] + - [int32x4_t, int8x16_t] + - [uint32x4_t, int8x16_t] + - [int64x2_t, int16x8_t] + - [uint64x2_t, int16x8_t] + - [int32x2_t, uint8x8_t] + - [uint32x2_t, uint8x8_t] + - [int64x1_t, uint16x4_t] + - [uint64x1_t, uint16x4_t] + - [int32x4_t, uint8x16_t] + - [uint32x4_t, uint8x16_t] + - [int64x2_t, uint16x8_t] + - [uint64x2_t, uint16x8_t] + - [int32x2_t, poly8x8_t] + - [uint32x2_t, poly8x8_t] + - [int64x1_t, poly16x4_t] + - [uint64x1_t, poly16x4_t] + - [int32x4_t, poly8x16_t] + - [uint32x4_t, poly8x16_t] + - [int64x2_t, poly16x8_t] + - [uint64x2_t, poly16x8_t] + - [poly8x8_t, int32x2_t] + - [int8x8_t, int32x2_t] + - [uint8x8_t, int32x2_t] + - [poly16x4_t, int64x1_t] + - [int16x4_t, int64x1_t] + - [uint16x4_t, int64x1_t] + - [poly8x16_t, int32x4_t] + - [int8x16_t, int32x4_t] + - [uint8x16_t, int32x4_t] + - [poly16x8_t, int64x2_t] + - [int16x8_t, int64x2_t] + - [uint16x8_t, int64x2_t] + - [poly8x8_t, uint32x2_t] + - [int8x8_t, uint32x2_t] + - [uint8x8_t, uint32x2_t] + - [poly16x4_t, uint64x1_t] + - [int16x4_t, uint64x1_t] + - [uint16x4_t, uint64x1_t] + - [poly8x16_t, uint32x4_t] + - [int8x16_t, uint32x4_t] + - [uint8x16_t, uint32x4_t] + - [poly16x8_t, uint64x2_t] + - [int16x8_t, uint64x2_t] + - [uint16x8_t, uint64x2_t] + - [int64x1_t, int8x8_t] + - [uint64x1_t, int8x8_t] + - [int64x1_t, uint8x8_t] + - [uint64x1_t, uint8x8_t] + - [int64x1_t, poly8x8_t] + - [uint64x1_t, poly8x8_t] + - [int64x2_t, int8x16_t] + - [uint64x2_t, int8x16_t] + - [int64x2_t, uint8x16_t] + - [uint64x2_t, uint8x16_t] + - [int64x2_t, poly8x16_t] + - [uint64x2_t, poly8x16_t] + - [poly8x8_t, int64x1_t] + - [int8x8_t, int64x1_t] + - [uint8x8_t, int64x1_t] + - [poly8x8_t, uint64x1_t] + - [int8x8_t, uint64x1_t] + - [uint8x8_t, uint64x1_t] + - [poly8x16_t, int64x2_t] + - [int8x16_t, int64x2_t] + - [uint8x16_t, int64x2_t] + - [poly8x16_t, uint64x2_t] + - [int8x16_t, uint64x2_t] + - [uint8x16_t, uint64x2_t] + - [float32x2_t, int8x8_t] + - [float32x2_t, int16x4_t] + - [float32x2_t, int32x2_t] + - [float32x2_t, int64x1_t] + - [float32x4_t, int8x16_t] + - [float32x4_t, int16x8_t] + - [float32x4_t, int32x4_t] + - [float32x4_t, int64x2_t] + - [float32x2_t, uint8x8_t] + - [float32x2_t, uint16x4_t] + - [float32x2_t, uint32x2_t] + - [float32x2_t, uint64x1_t] + - [float32x4_t, uint8x16_t] + - [float32x4_t, uint16x8_t] + - [float32x4_t, uint32x4_t] + - [float32x4_t, uint64x2_t] + - [float32x2_t, poly8x8_t] + - [float32x2_t, poly16x4_t] + - [float32x4_t, poly8x16_t] + - [float32x4_t, poly16x8_t] + - [float32x4_t, p128] + - [int8x8_t, float32x2_t] + - [int16x4_t, float32x2_t] + - [int32x2_t, float32x2_t] + - [int64x1_t, float32x2_t] + - [int8x16_t, float32x4_t] + - [int16x8_t, float32x4_t] + - [int32x4_t, float32x4_t] + - [int64x2_t, float32x4_t] + - [uint8x8_t, float32x2_t] + - [uint16x4_t, float32x2_t] + - [uint32x2_t, float32x2_t] + - [uint64x1_t, float32x2_t] + - [uint8x16_t, float32x4_t] + - [uint16x8_t, float32x4_t] + - [uint32x4_t, float32x4_t] + - [uint64x2_t, float32x4_t] + - [poly8x8_t, float32x2_t] + - [poly16x4_t, float32x2_t] + - [poly8x16_t, float32x4_t] + - [poly16x8_t, float32x4_t] + - [p128, float32x4_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vrshl{neon_type.no}" + doc: "Signed rounding shift left" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "vrshl{neon_type.no}" + links: + - link: "llvm.arm.neon.vrshifts.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.srshl.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrshl{neon_type[0].no}" + doc: "Unsigned rounding shift left" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [uint16x4_t, int16x4_t] + - [uint16x8_t, int16x8_t] + - [uint32x2_t, int32x2_t] + - [uint32x4_t, int32x4_t] + - [uint64x1_t, int64x1_t] + - [uint64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vrshl{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vrshiftu.{neon_type[0]}" + arch: arm + - link: "llvm.aarch64.neon.urshl.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vrshr{neon_type[0].N}" + doc: "Signed rounding shift right" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshr, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, 'N >= 1 && N <= 8'] + - [int8x16_t, 'N >= 1 && N <= 8'] + - [int16x4_t, 'N >= 1 && N <= 16'] + - [int16x8_t, 'N >= 1 && N <= 16'] + - [int32x2_t, 'N >= 1 && N <= 32'] + - [int32x4_t, 'N >= 1 && N <= 32'] + - [int64x1_t, 'N >= 1 && N <= 64'] + - [int64x2_t, 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - FnCall: + - "vrshl{neon_type[0].no}" + - - a + - FnCall: ["vdup{neon_type[0].N}", ['-N as _']] + + - name: "vrshr{neon_type[0].N}" + doc: "Unsigned rounding shift right" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshr, N = 2]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t, 'N >= 1 && N <= 8'] + - [uint8x16_t, int8x16_t, 'N >= 1 && N <= 8'] + - [uint16x4_t, int16x4_t, 'N >= 1 && N <= 16'] + - [uint16x8_t, int16x8_t, 'N >= 1 && N <= 16'] + - [uint32x2_t, int32x2_t, 'N >= 1 && N <= 32'] + - [uint32x4_t, int32x4_t, 'N >= 1 && N <= 32'] + - [uint64x1_t, int64x1_t, 'N >= 1 && N <= 64'] + - [uint64x2_t, int64x2_t, 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - FnCall: + - "vrshl{neon_type[0].no}" + - - a + - FnCall: ["vdup{neon_type[1].N}", ['-N as _']] + + - name: "vrshrn_n_{neon_type[0]}" + doc: "Rounding shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vrshrn_n_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vrshiftn.{neon_type[1]}" + arch: arm + - FnCall: ["_vrshrn_n_{neon_type[0]}", [a, "{type[3]}"]] + + - name: "vrshrn_n_{neon_type[0]}" + doc: "Rounding shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] + - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] + - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - LLVMLink: + name: "vrshrn_n_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.rshrn.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vrshrn_n_{neon_type[0]}", [a, N]] + + - name: "vrshrn_n_{neon_type[0]}" + doc: "Rounding shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshrn, N = 2]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8', s16] + - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16', s32] + - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32', s64] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - FnCall: + - transmute + - - FnCall: + - "vrshrn_n_{type[3]}::" + - - FnCall: [transmute, [a]] + + - name: "vrsra{neon_type[0].N}" + doc: "Signed rounding shift right and accumulate" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsra, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srsra, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, 'N >= 1 && N <= 8'] + - [int8x16_t, 'N >= 1 && N <= 8'] + - [int16x4_t, 'N >= 1 && N <= 16'] + - [int16x8_t, 'N >= 1 && N <= 16'] + - [int32x2_t, 'N >= 1 && N <= 32'] + - [int32x4_t, 'N >= 1 && N <= 32'] + - [int64x1_t, 'N >= 1 && N <= 64'] + - [int64x2_t, 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - FnCall: + - simd_add + - - a + - FnCall: ["vrshr{neon_type[0].N}::", [b]] + + - name: "vrsubhn_{neon_type[0]}" + doc: "Rounding subtract returning high narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsubhn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rsubhn]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int16x8_t, int8x8_t] + - [int32x4_t, int32x4_t, int16x4_t] + - [int64x2_t, int64x2_t, int32x2_t] + compose: + - LLVMLink: + name: "vrsubhn_{neon_type[0]}" + links: + - link: "llvm.arm.neon.vrsubhn.{neon_type[2]}" + arch: arm + - link: "llvm.aarch64.neon.rsubhn.{neon_type[2]}" + arch: aarch64,arm64ec + + - name: "vrsubhn_{neon_type[0]}" + doc: "Rounding subtract returning high narrow" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsubhn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rsubhn]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint16x8_t, uint8x8_t, s16] + - [uint32x4_t, uint32x4_t, uint16x4_t, s32] + - [uint64x2_t, uint64x2_t, uint32x2_t, s64] + compose: + - FnCall: + - transmute + - - FnCall: + - "vrsubhn_{type[3]}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vcreate_{neon_type[1]}" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["u64", int8x8_t] + - ["u64", int16x4_t] + - ["u64", int32x2_t] + - ["u64", int64x1_t] + - ["u64", uint8x8_t] + - ["u64", uint16x4_t] + - ["u64", uint32x2_t] + - ["u64", uint64x1_t] + - ["u64", poly8x8_t] + - ["u64", poly16x4_t] + - ["u64", float32x2_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vcreate_p64" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["u64", poly64x1_t] + compose: + - FnCall: [transmute, [a]] + + - name: "vset{neon_type[1].lane_nox}" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, LANE = 0]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i8", int8x8_t, '3'] + - ["i16", int16x4_t, '2'] + - ["i32", int32x2_t, '1'] + - ["u8", uint8x8_t, '3'] + - ["u16", uint16x4_t, '2'] + - ["u32", uint32x2_t, '1'] + - ["p8", poly8x8_t, '3'] + - ["p16", poly16x4_t, '2'] + - ["i8", int8x16_t, '4'] + - ["i16", int16x8_t, '3'] + - ["i32", int32x4_t, '2'] + - ["i64", int64x2_t, '1'] + - ["u8", uint8x16_t, '4'] + - ["u16", uint16x8_t, '3'] + - ["u32", uint32x4_t, '2'] + - ["u64", uint64x2_t, '1'] + - ["p8", poly8x16_t, '4'] + - ["p16", poly16x8_t, '3'] + - ["f32", float32x2_t, '1'] + - ["f32", float32x4_t, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vset_lane_{neon_type[0]}" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["i64", int64x1_t, int64x1_t] + - ["u64", uint64x1_t, uint64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vset_lane_{neon_type[0]}" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["p64", poly64x1_t, poly64x1_t] + compose: + - FnCall: [static_assert!, ['LANE == 0']] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vsetq_lane_p64" + doc: "Insert vector element from another vector element" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - ["p64", poly64x2_t, poly64x2_t] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '1']] + - FnCall: [simd_insert!, [b, 'LANE as u32', a]] + + - name: "vshl{neon_type.no}" + doc: "Signed Shift left" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + - int64x1_t + - int64x2_t + compose: + - LLVMLink: + name: "vshl{neon_type.no}" + links: + - link: "llvm.arm.neon.vshifts.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.sshl.{neon_type}" + arch: aarch64,arm64ec + + - name: "vshl{neon_type[0].no}" + doc: "Unsigned Shift left" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, int8x8_t] + - [uint8x16_t, int8x16_t] + - [uint16x4_t, int16x4_t] + - [uint16x8_t, int16x8_t] + - [uint32x2_t, int32x2_t] + - [uint32x4_t, int32x4_t] + - [uint64x1_t, int64x1_t] + - [uint64x2_t, int64x2_t] + compose: + - LLVMLink: + name: "vshl{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vshiftu.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.ushl.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vshll_n_s8" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s8"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, int16x8_t, 'N >= 0 && N <= 8'] + compose: + - FnCall: [static_assert!, ["{type[2]}"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_s16, ['N as _']] + + - name: "vshll_n_s16" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s16"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, int32x4_t] + compose: + - FnCall: [static_assert!, ["N >= 0 && N <= 16"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_s32, ['N as _']] + + - name: "vshll_n_s32" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s32"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int32x2_t, int64x2_t] + compose: + - FnCall: [static_assert!, ["N >= 0 && N <= 32"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_s64, ['N as _']] + + - name: "vshll_n_u8" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u8"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint16x8_t] + compose: + - FnCall: [static_assert!, ["N >= 0 && N <= 8"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_u16, ['N as _']] + + - name: "vshll_n_u16" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u16"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x4_t, uint32x4_t] + compose: + - FnCall: [static_assert!, ["N >= 0 && N <= 16"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_u32, ['N as _']] + + - name: "vshll_n_u32" + doc: "Signed shift left long" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u32"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint32x2_t, uint64x2_t] + compose: + - FnCall: [static_assert!, ["N >= 0 && N <= 32"]] + - FnCall: + - simd_shl + - - FnCall: [simd_cast, [a]] + - FnCall: [vdupq_n_u64, ['N as _']] + + - name: "vshr{neon_type[0].N}" + doc: "Shift right" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshr.{neon_type[0]}"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { 7 }', 'else { N };'] + - [int8x16_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { 7 }', 'else { N };'] + - [int16x4_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { 15 }', 'else { N };'] + - [int16x8_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { 15 }', 'else { N };'] + - [int32x2_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { 31 }', 'else { N };'] + - [int32x4_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { 31 }', 'else { N };'] + - [int64x1_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { 63 }', 'else { N };'] + - [int64x2_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { 63 }', 'else { N };'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - Identifier: ["{type[2]}{type[3]}", Symbol] + - FnCall: + - simd_shr + - - a + - FnCall: ["vdup{neon_type[0].N}", ['n as _']] + + - name: "vshr{neon_type[0].N}" + doc: "Shift right" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshr.{neon_type[0]}"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushr, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { return vdup_n_u8(0); }', 'else { N };'] + - [uint8x16_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { return vdupq_n_u8(0); }', 'else { N };'] + - [uint16x4_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { return vdup_n_u16(0); }', 'else { N };'] + - [uint16x8_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { return vdupq_n_u16(0); }', 'else { N };'] + - [uint32x2_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { return vdup_n_u32(0); }', 'else { N };'] + - [uint32x4_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { return vdupq_n_u32(0); }', 'else { N };'] + - [uint64x1_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { return vdup_n_u64(0); }', 'else { N };'] + - [uint64x2_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { return vdupq_n_u64(0); }', 'else { N };'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - Identifier: ['{type[2]}{type[3]}', Symbol] + - FnCall: + - simd_shr + - - a + - FnCall: ["vdup{neon_type[0].N}", ['n as _']] + + - name: "vshrn_n_{neon_type[0]}" + doc: "Shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshrn{type[2]}"', 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [shrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, '.i16', 'N >= 1 && N <= 8'] + - [uint16x8_t, uint8x8_t, '.i16', 'N >= 1 && N <= 8'] + - [int32x4_t, int16x4_t, '.i32', 'N >= 1 && N <= 16'] + - [uint32x4_t, uint16x4_t, '.i32', 'N >= 1 && N <= 16'] + - [int64x2_t, int32x2_t, '.i64', 'N >= 1 && N <= 32'] + - [uint64x2_t, uint32x2_t, '.i64', 'N >= 1 && N <= 32'] + compose: + - FnCall: [static_assert!, ["{type[3]}"]] + - FnCall: + - simd_cast + - - FnCall: + - simd_shr + - - a + - FnCall: ["vdupq_n_{neon_type[0]}", ['N as _']] + + - name: "vsra{neon_type[0].N}" + doc: "Signed shift right and accumulate" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vsra, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ssra, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, 'N >= 1 && N <= 8'] + - [int8x16_t, 'N >= 1 && N <= 8'] + - [int16x4_t, 'N >= 1 && N <= 16'] + - [int16x8_t, 'N >= 1 && N <= 16'] + - [int32x2_t, 'N >= 1 && N <= 32'] + - [int32x4_t, 'N >= 1 && N <= 32'] + - [int64x1_t, 'N >= 1 && N <= 64'] + - [int64x2_t, 'N >= 1 && N <= 64'] + compose: + - FnCall: [static_assert!, ["{type[1]}"]] + - FnCall: + - simd_add + - - a + - FnCall: ["vshr{neon_type[0].N}::", [b]] + + - name: "vtrn{neon_type[0].no}" + doc: "Transpose elements" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtrn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [trn]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int8x8_t, int8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [int16x4_t, int16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + - [int8x16_t, int8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [int16x8_t, int16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [int32x4_t, int32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + - [uint8x8_t, uint8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [uint16x4_t, uint16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + - [uint8x16_t, uint8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [uint16x8_t, uint16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [uint32x4_t, uint32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + - [poly8x8_t, poly8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [poly16x4_t, poly16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + - [poly8x16_t, poly8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] + - [poly16x8_t, poly16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] + - [float32x4_t, float32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] + compose: + - Let: + - a1 + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [a, b, "{type[2]}"]] + - Let: + - b1 + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - Identifier: ['(a1, b1)', Symbol] + + - name: "vtrn{neon_type[0].no}" + doc: "Transpose elements" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtrn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [zip]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] + - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] + - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] + compose: + - Let: + - a1 + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [a, b, "{type[2]}"]] + - Let: + - b1 + - "{neon_type[0]}" + - FnCall: [simd_shuffle!, [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - Identifier: ['(a1, b1)', Symbol] + + - name: "vzip{neon_type[0].no}" + doc: Zip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vorr]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x16_t, int8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [int16x8_t, int16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [int32x4_t, int32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + - [uint8x16_t, uint8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [uint16x8_t, uint16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [uint32x4_t, uint32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + - [poly8x16_t, poly8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] + - [poly16x8_t, poly16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [float32x4_t, float32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + compose: + - Let: + - a0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] + - Let: + - b0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - '(a0, b0)' + + - name: "vzip{neon_type[0].no}" + doc: Zip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vtrn]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] + - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] + - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] + compose: + - Let: + - a0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] + - Let: + - b0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - '(a0, b0)' + + - name: "vzip{neon_type[0].no}" + doc: Zip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vzip]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, int8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [int16x4_t, int16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + - [uint8x8_t, uint8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [uint16x4_t, uint16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + - [poly8x8_t, poly8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] + - [poly16x4_t, poly16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] + compose: + - Let: + - a0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] + - Let: + - b0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - '(a0, b0)' + + - name: "vuzp{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vuzp]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uzp]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [int8x8_t, int8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [int16x4_t, int16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + - [int8x16_t, int8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [int16x8_t, int16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [int32x4_t, int32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + - [uint8x8_t, uint8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [uint16x4_t, uint16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + - [uint8x16_t, uint8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [uint16x8_t, uint16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [uint32x4_t, uint32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + - [poly8x8_t, poly8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [poly16x4_t, poly16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + - [poly8x16_t, poly8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] + - [poly16x8_t, poly16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] + - [float32x4_t, float32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] + compose: + - Let: + - a0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] + - Let: + - b0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - '(a0, b0)' + + - name: "vuzp{neon_type[0].no}" + doc: Unzip vectors + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vtrn]]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] + - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] + - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] + compose: + - Let: + - a0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] + - Let: + - b0 + - "{neon_type[0]}" + - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] + - FnCall: + - transmute + - - '(a0, b0)' + + - name: "vabal_{neon_type[1]}" + doc: "Unsigned Absolute difference and Accumulate Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabal.{type[2]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uabal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, "u8"] + - [uint32x4_t, uint16x4_t, "u16"] + - [uint64x2_t, uint32x2_t, "u32"] + compose: + - Let: [d, "{neon_type[1]}", {FnCall: ["vabd_{type[2]}", [b, c]]}] + - FnCall: [simd_add, [a, {FnCall: [simd_cast, [d]]}]] + + - name: "vabal_{neon_type[1]}" + doc: "Signed Absolute difference and Accumulate Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabal.{neon_type[1]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sabal]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x8_t, int8x8_t, uint8x8_t] + - [int32x4_t, int16x4_t, uint16x4_t] + - [int64x2_t, int32x2_t, uint32x2_t] + compose: + - Let: [d, "{type[1]}", {FnCall: ["vabd_{neon_type[1]}", [b, c]]}] + - Let: [e, "{type[2]}", {FnCall: ["simd_cast", [d]]}] + - FnCall: [simd_add, [a, {FnCall: [simd_cast, [e]]}]] + + - name: "vqabs{neon_type.no}" + doc: Signed saturating Absolute value + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - *neon-v7 + - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqabs.{neon_type}"']]}]] + - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqabs]]}]] + - *neon-stable-not-arm + - *neon-unstable-is-arm + safety: + unsafe: [neon] + types: + - int8x8_t + - int8x16_t + - int16x4_t + - int16x8_t + - int32x2_t + - int32x4_t + compose: + - LLVMLink: + name: "sqabs.{neon_type}" + links: + - link: "llvm.aarch64.neon.sqabs.{neon_type}" + arch: aarch64,arm64ec + - link: "llvm.arm.neon.vqabs.{neon_type}" + arch: arm + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vst1]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*mut u8", uint8x8x2_t, int8x8x2_t] + - ["*mut u16", uint16x4x2_t, int16x4x2_t] + - ["*mut u32", uint32x2x2_t, int32x2x2_t] + - ["*mut u64", uint64x1x2_t, int64x1x2_t] + - ["*mut u8", uint8x16x2_t, int8x16x2_t] + - ["*mut u16", uint16x8x2_t, int16x8x2_t] + - ["*mut u32", uint32x4x2_t, int32x4x2_t] + - ["*mut u64", uint64x2x2_t, int64x2x2_t] + - ["*mut u8", uint8x8x3_t, int8x8x3_t] + - ["*mut u16", uint16x4x3_t, int16x4x3_t] + - ["*mut u32", uint32x2x3_t, int32x2x3_t] + - ["*mut u64", uint64x1x3_t, int64x1x3_t] + - ["*mut u8", uint8x16x3_t, int8x16x3_t] + - ["*mut u16", uint16x8x3_t, int16x8x3_t] + - ["*mut u32", uint32x4x3_t, int32x4x3_t] + - ["*mut u64", uint64x2x3_t, int64x2x3_t] + - ["*mut u8", uint8x8x4_t, int8x8x4_t] + - ["*mut u16", uint16x4x4_t, int16x4x4_t] + - ["*mut u32", uint32x2x4_t, int32x2x4_t] + - ["*mut u64", uint64x1x4_t, int64x1x4_t] + - ["*mut u8", uint8x16x4_t, int8x16x4_t] + - ["*mut u16", uint16x8x4_t, int16x8x4_t] + - ["*mut u32", uint32x4x4_t, int32x4x4_t] + - ["*mut u64", uint64x2x4_t, int64x2x4_t] + - ["*mut p8", poly8x8x2_t, int8x8x2_t] + - ["*mut p8", poly8x8x3_t, int8x8x3_t] + - ["*mut p8", poly8x8x4_t, int8x8x4_t] + - ["*mut p8", poly8x16x2_t, int8x16x2_t] + - ["*mut p8", poly8x16x3_t, int8x16x3_t] + - ["*mut p8", poly8x16x4_t, int8x16x4_t] + - ["*mut p16", poly16x4x2_t, int16x4x2_t] + - ["*mut p16", poly16x4x3_t, int16x4x3_t] + - ["*mut p16", poly16x4x4_t, int16x4x4_t] + - ["*mut p16", poly16x8x2_t, int16x8x2_t] + - ["*mut p16", poly16x8x3_t, int16x8x3_t] + - ["*mut p16", poly16x8x4_t, int16x8x4_t] + compose: + - FnCall: + - "vst1{neon_type[2].no}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vst1]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*mut p64", poly64x1x2_t, int64x1x2_t] + compose: + - FnCall: + - "vst1{neon_type[2].no}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*mut p64", poly64x1x3_t, int64x1x3_t] + - ["*mut p64", poly64x1x4_t, int64x1x4_t] + - ["*mut p64", poly64x2x2_t, int64x2x2_t] + - ["*mut p64", poly64x2x3_t, int64x2x3_t] + - ["*mut p64", poly64x2x4_t, int64x2x4_t] + compose: + - FnCall: + - "vst1{neon_type[2].no}" + - - FnCall: [transmute, [a]] + - FnCall: [transmute, [b]] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + safety: + unsafe: [neon] + types: + - ['*mut f32', float32x2x2_t, float32x2_t] + - ['*mut f32', float32x4x2_t, float32x4_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "ptr: {type[0]}" + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + links: + - link: "llvm.arm.neon.vst1x{neon_type[1].tuple}.p0f32.{neon_type[2]}" + arch: arm + - FnCall: ["_vst1{neon_type[1].no}", ['a', 'b.0', 'b.1']] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + safety: + unsafe: [neon] + types: + - ['*mut f32', float32x2x3_t, float32x2_t] + - ['*mut f32', float32x4x3_t, float32x4_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "ptr: {type[0]}" + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "c: {neon_type[2]}" + links: + - link: "llvm.arm.neon.vst1x{neon_type[1].tuple}.p0f32.{neon_type[2]}" + arch: arm + - FnCall: ["_vst1{neon_type[1].no}", ['a', 'b.0', 'b.1', 'b.2']] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*mut f32", float32x2x2_t, float32x2_t] + - ["*mut f32", float32x4x2_t, float32x4_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x2.{neon_type[2]}.p0f32" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', a]] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*mut f32", float32x2x3_t, float32x2_t] + - ["*mut f32", float32x4x3_t, float32x4_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "c: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x3.{neon_type[2]}.p0f32" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', a]] + + - name: "vst1{neon_type[1].no}" + doc: "Store multiple single-element structures to one, two, three, or four registers" + arguments: ["a: {type[0]}", "b: {neon_type[1]}"] + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*mut f32", float32x2x4_t, float32x2_t] + - ["*mut f32", float32x4x4_t, float32x4_t] + compose: + - LLVMLink: + name: "vst1{neon_type[1].no}" + arguments: + - "a: {neon_type[2]}" + - "b: {neon_type[2]}" + - "c: {neon_type[2]}" + - "d: {neon_type[2]}" + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.st1x4.{neon_type[2]}.p0f32" + arch: aarch64,arm64ec + - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'b.3', a]] + + - name: "vfms{neon_type.no}" + doc: "Floating-point fused multiply-subtract from accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vfms]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] + - FnCall: ["vfma{neon_type.no}", [a, b, c]] + + - name: "vmul{neon_type[0].no}" + doc: "Polynomial multiply" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [pmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [poly8x8_t, int8x8_t] + - [poly8x16_t, int8x16_t] + compose: + - LLVMLink: + name: "vmul{neon_type[0].no}" + links: + - link: "llvm.arm.neon.vmulp.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.pmul.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vmls{neon_type.no}" + doc: "Floating-point multiply-subtract from accumulator" + arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - FnCall: [simd_sub, [a, {FnCall: [simd_mul, [b, c]]}]] + + - name: "vcge{neon_type.no}" + doc: "Compare unsigned greater than or equal" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhs]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vcge{neon_type[0].no}" + doc: "Floating-point compare greater than or equal" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmge]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: [simd_ge, [a, b]] + + - name: "vclt{neon_type.no}" + doc: "Compare unsigned less than" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhi]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vtst{neon_type[0].no}" + doc: "Unsigned compare bitwise Test bits nonzero" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtst]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmtst]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, u8x8, 'u8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint8x16_t, u8x16, 'u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint16x4_t, u16x4, 'u16x4::new(0, 0, 0, 0)'] + - [uint16x8_t, u16x8, 'u16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] + - [uint32x2_t, u32x2, 'u32x2::new(0, 0)'] + - [uint32x4_t, u32x4, 'u32x4::new(0, 0, 0, 0)'] + compose: + - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] + - Let: [d, "{type[1]}", "{type[2]}"] + - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] + + - name: "vshl{neon_type[0].N}" + doc: "Shift left" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [shl, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, '3'] + - [int8x16_t, '3'] + - [int16x4_t, '4'] + - [int16x8_t, '4'] + - [int32x2_t, '5'] + - [int32x4_t, '5'] + - [uint8x8_t, '3'] + - [uint8x16_t, '3'] + - [uint16x4_t, '4'] + - [uint16x8_t, '4'] + - [uint32x2_t, '5'] + - [uint32x4_t, '5'] + - [int64x1_t, '6'] + - [int64x2_t, '6'] + - [uint64x1_t, '6'] + - [uint64x2_t, '6'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] + - FnCall: + - simd_shl + - - a + - FnCall: ["vdup{neon_type[0].N}", ['N as _']] + + - name: "vsra{neon_type[0].N}" + doc: "Unsigned shift right and accumulate" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vsra, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [usra, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, '8'] + - [uint8x16_t, '8'] + - [uint16x4_t, '16'] + - [uint16x8_t, '16'] + - [uint32x2_t, '32'] + - [uint32x4_t, '32'] + - [uint64x1_t, '64'] + - [uint64x2_t, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[1]}']] + - FnCall: + - simd_add + - - a + - FnCall: ["vshr{neon_type[0].N}::", [b]] + + - name: "vrsra{neon_type[0].N}" + doc: "Unsigned rounding shift right and accumulate" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsra, 'N = 2']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ursra, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint8x8_t, '8'] + - [uint8x16_t, '8'] + - [uint16x4_t, '16'] + - [uint16x8_t, '16'] + - [uint32x2_t, '32'] + - [uint32x4_t, '32'] + - [uint64x1_t, '64'] + - [uint64x2_t, '64'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[1]}']] + - FnCall: + - simd_add + - - a + - FnCall: ["vrshr{neon_type[0].N}::", [b]] + + - name: "vqrshrn_n_{neon_type[0]}" + doc: "Unsigned signed saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, '8', 'const { uint16x8_t([-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16]) }.as_signed()'] + - [uint32x4_t, uint16x4_t, '16', 'const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }.as_signed()'] + - [uint64x2_t, uint32x2_t, '32', 'const { uint64x2_t([-N as u64, -N as u64]) }.as_signed()'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[2]}']] + - LLVMLink: + name: "vqrshrn{neon_type[0].N}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqrshiftnu.{neon_type[1]}" + arch: arm + - FnCall: ["_vqrshrn_n{neon_type[0].noq}", ["a.as_signed()", "{type[3]}"]] + + - name: "vqrshrn_n_{neon_type[0]}" + doc: "Unsigned signed saturating rounded shift right narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-not-arm + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t, '8'] + - [uint32x4_t, uint16x4_t, '16'] + - [uint64x2_t, uint32x2_t, '32'] + compose: + - FnCall: [static_assert!, ['N >= 1 && N <= {type[2]}']] + - LLVMLink: + name: "vqrshrn_n_{neon_type[0]}" + arguments: + - "a: {neon_type[0]}" + - "n: i32" + links: + - link: "llvm.aarch64.neon.uqrshrn.{neon_type[1]}" + arch: aarch64,arm64ec + - FnCall: ["_vqrshrn_n_{neon_type[0]}", ["a.as_signed()", N]] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to unsigned fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcvtzu]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - LLVMLink: + name: "vcvt{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" + arch: arm + - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vmla{neon_type[0].N}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i16"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", int16x4_t] + - [int16x8_t, "i16", int16x8_t] + - [uint16x4_t, "u16", uint16x4_t] + - [uint16x8_t, "u16", uint16x8_t] + compose: + - FnCall: + - "vmla{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [c]] + + - name: "vmla{neon_type[0].N}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, "i32", int32x2_t] + - [int32x4_t, "i32", int32x4_t] + - [uint32x2_t, "u32", uint32x2_t] + - [uint32x4_t, "u32", uint32x4_t] + compose: + - FnCall: + - "vmla{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [c]] + + - name: "vmla{neon_type[0].N}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, "f32", float32x2_t] + - [float32x4_t, "f32", float32x4_t] + compose: + - FnCall: ["vmla{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [c]]}]] + + - name: "vmla{type[0]}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i16"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_u16, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_u16, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmla{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmla{type[0]}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i32"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_u32, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_u32, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmla{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmla{type[0]}" + doc: "Vector multiply accumulate with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_f32, float32x2_t, float32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_f32, float32x2_t, float32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_f32, float32x4_t, float32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmla{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmls{neon_type[0].N}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i16"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16", int16x4_t] + - [int16x8_t, "i16", int16x8_t] + - [uint16x4_t, "u16", uint16x4_t] + - [uint16x8_t, "u16", uint16x8_t] + compose: + - FnCall: + - "vmls{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [c]] + + - name: "vmls{neon_type[0].N}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int32x2_t, "i32", int32x2_t] + - [int32x4_t, "i32", int32x4_t] + - [uint32x2_t, "u32", uint32x2_t] + - [uint32x4_t, "u32", uint32x4_t] + compose: + - FnCall: + - "vmls{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}", [c]] + + - name: "vmls{neon_type[0].N}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, "f32", float32x2_t] + - [float32x4_t, "f32", float32x4_t] + compose: + - FnCall: ["vmls{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [c]]}]] + + - name: "vmls{type[0]}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i16"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_u16, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_u16, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmls{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmls{type[0]}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i32"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_u32, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_u32, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmls{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmls{type[0]}" + doc: "Vector multiply subtract with scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"', 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['3']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_f32, float32x2_t, float32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_f32, float32x2_t, float32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_f32, float32x4_t, float32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - "vmls{neon_type[1].no}" + - - a + - b + - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] + + - name: "vmul{neon_type[0].N}" + doc: "Vector multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16"] + - [int16x8_t, "i16"] + - [int32x2_t, "i32"] + - [int32x4_t, "i32"] + - [uint16x4_t, "u16"] + - [uint16x8_t, "u16"] + - [uint32x2_t, "u32"] + - [uint32x4_t, "u32"] + compose: + - FnCall: + - simd_mul + - - a + - FnCall: ["vdup{neon_type[0].N}", [b]] + + - name: "vmul{neon_type[0].N}" + doc: "Vector multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, "f32"] + - [float32x4_t, "f32"] + compose: + - FnCall: + - simd_mul + - - a + - FnCall: ["vdup{neon_type[0].N}", [b]] + + - name: "vmul{type[2]}" + doc: "Floating-point multiply" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [float32x2_t, float32x2_t, '_lane_f32', '1', '[LANE as u32, LANE as u32]'] + - [float32x2_t, float32x4_t, '_laneq_f32', '2', '[LANE as u32, LANE as u32]'] + - [float32x4_t, float32x2_t, 'q_lane_f32', '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [float32x4_t, float32x4_t, 'q_laneq_f32', '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] + - FnCall: + - simd_mul + - - a + - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] + + - name: "vqrdmulh{type[0]}" + doc: "Vector rounding saturating doubling multiply high by scalar" + arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh, 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] + - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] + - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - Let: [b, "{neon_type[1]}", {FnCall: [simd_shuffle!, [b, b, '{type[4]}']]}] + - FnCall: ["vqrdmulh{neon_type[1].no}", [a, b]] + + - name: "vqrdmulh{neon_type[0].N}" + doc: "Vector saturating rounding doubling multiply high with scalar" + arguments: ["a: {neon_type[0]}", "b: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [int16x4_t, "i16"] + - [int16x8_t, "i16"] + - [int32x2_t, "i32"] + - [int32x4_t, "i32"] + compose: + - FnCall: + - "vqrdmulh{neon_type[0].no}" + - - a + - FnCall: ["vdup{neon_type[0].N}", [b]] + + - name: "vclt{neon_type[0].no}" + doc: "Floating-point compare less than" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.f32"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmgt]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, uint32x2_t] + - [float32x4_t, uint32x4_t] + compose: + - FnCall: [simd_lt, [a, b]] + + - name: "vabdl_{neon_type[0]}" + doc: "Unsigned Absolute difference Long" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabdl.{neon_type[0]}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uabdl]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint8x8_t, uint16x8_t] + - [uint16x4_t, uint32x4_t] + - [uint32x2_t, uint64x2_t] + compose: + - FnCall: [simd_cast, [{FnCall: ["vabd_{neon_type[0]}", [a, b]]}]] + + - name: "vmull_lane{neon_type[1].no}" + doc: "Vector long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmull, 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smull, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int16x4_t, int16x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int16x4_t, int16x8_t, int32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [int32x2_t, int32x2_t, int64x2_t, '1', '[LANE as u32, LANE as u32]'] + - [int32x2_t, int32x4_t, int64x2_t, '2', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmull_{neon_type[0]}" + - - a + - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] + + - name: "vmull_lane{neon_type[1].no}" + doc: "Vector long multiply by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[2]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmull, 'LANE = 1']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umull, 'LANE = 1']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [uint16x4_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint16x4_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] + - [uint32x2_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32]'] + - [uint32x2_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32]'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] + - FnCall: + - "vmull_{neon_type[0]}" + - - a + - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] + + - name: "vfms{neon_type[0].N}" + doc: "Floating-point fused Multiply-subtract to accumulator(vector)" + arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vfms]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmls]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, "f32"] + - [float32x4_t, "f32"] + compose: + - FnCall: + - "vfms{neon_type[0].no}" + - - a + - b + - FnCall: ["vdup{neon_type[0].N}_vfp4", [c]] + + - name: "vqdmulh{neon_type[0].laneq_nox}" + doc: "Vector saturating doubling multiply high by scalar" + arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] + return_type: "{neon_type[0]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh, 'LANE = 0']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh, 'LANE = 0']]}]] + - FnCall: [rustc_legacy_const_generics, ['2']] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + static_defs: ['const LANE: i32'] + safety: + unsafe: [neon] + types: + - [int16x8_t, int16x8_t, '3'] + - [int16x4_t, int16x8_t, '3'] + - [int32x4_t, int32x4_t, '2'] + - [int32x2_t, int32x4_t, '2'] + compose: + - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] + - FnCall: + - "vqdmulh{neon_type[0].no}" + - - a + - FnCall: + - "vdup{neon_type[0].N}" + - - FnCall: [simd_extract!, [b, 'LANE as u32']] + + - name: "vrecpe{neon_type.no}" + doc: "Unsigned reciprocal estimate" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecpe]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urecpe]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "vrecpe{neon_type.no}" + links: + - link: "llvm.arm.neon.vrecpe.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.urecpe.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrsqrte{neon_type.no}" + doc: "Unsigned reciprocal square root estimate" + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrte]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ursqrte]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint32x2_t + - uint32x4_t + compose: + - LLVMLink: + name: "vrsqrte{neon_type.no}" + links: + - link: "llvm.arm.neon.vrsqrte.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.ursqrte.{neon_type}" + arch: aarch64,arm64ec + + - name: "vrsqrte{neon_type.no}" + doc: "Reciprocal square-root estimate." + arguments: ["a: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrte]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frsqrte]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - float32x2_t + - float32x4_t + compose: + - LLVMLink: + name: "vrsqrte{neon_type.no}" + links: + - link: "llvm.arm.neon.vrsqrte.{neon_type}" + arch: arm + - link: "llvm.aarch64.neon.frsqrte.{neon_type}" + arch: aarch64,arm64ec + + - name: "vqshlu{neon_type[0].N}" + doc: "Signed saturating shift left unsigned" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - *target-is-arm + - *enable-v7 + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshlu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, '3', 'const { int8x8_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] + - [int16x4_t, uint16x4_t, '4', 'const { int16x4_t([N as i16, N as i16, N as i16, N as i16]) }'] + - [int32x2_t, uint32x2_t, '5', 'const { int32x2_t([N as i32, N as i32]) }'] + - [int64x1_t, uint64x1_t, '6', 'const { int64x1_t([N as i64]) }'] + - [int8x16_t, uint8x16_t, '3', 'const { int8x16_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] + - [int16x8_t, uint16x8_t, '4', 'const { int16x8_t([N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16]) }'] + - [int32x4_t, uint32x4_t, '5', 'const { int32x4_t([N as i32, N as i32, N as i32, N as i32]) }'] + - [int64x2_t, uint64x2_t, '6', 'const { int64x2_t([N as i64, N as i64]) }'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] + - LLVMLink: + name: "vqshlu{neon_type[0].N}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.arm.neon.vqshiftsu.{neon_type[0]}" + arch: arm + - FnCall: ["_vqshlu{neon_type[0].N}", [a, "{type[3]}"]] + + - name: "vqshlu{neon_type[0].N}" + doc: "Signed saturating shift left unsigned" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshlu, 'N = 2']]}]] + - FnCall: [rustc_legacy_const_generics, ['1']] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + static_defs: ['const N: i32'] + safety: + unsafe: [neon] + types: + - [int8x8_t, uint8x8_t, '3', 'const { int8x8_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] + - [int16x4_t, uint16x4_t, '4', 'const { int16x4_t([N as i16, N as i16, N as i16, N as i16]) }'] + - [int32x2_t, uint32x2_t, '5', 'const { int32x2_t([N as i32, N as i32]) }'] + - [int64x1_t, uint64x1_t, '6', 'const { int64x1_t([N as i64]) }'] + - [int8x16_t, uint8x16_t, '3', 'const { int8x16_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] + - [int16x8_t, uint16x8_t, '4', 'const { int16x8_t([N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16]) }'] + - [int32x4_t, uint32x4_t, '5', 'const { int32x4_t([N as i32, N as i32, N as i32, N as i32]) }'] + - [int64x2_t, uint64x2_t, '6', 'const { int64x2_t([N as i64, N as i64]) }'] + compose: + - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] + - LLVMLink: + name: "vqshlu{neon_type[0].N}" + arguments: + - "a: {neon_type[0]}" + - "n: {neon_type[0]}" + links: + - link: "llvm.aarch64.neon.sqshlu.{neon_type[0]}" + arch: aarch64,arm64ec + - FnCall: ["_vqshlu{neon_type[0].N}", [a, "{type[3]}"]] + + - name: "vcvt{neon_type[1].no}_{neon_type[0]}" + doc: "Floating-point convert to signed fixed-point, rounding toward zero" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcvtzs]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [float32x2_t, int32x2_t] + - [float32x4_t, int32x4_t] + compose: + - LLVMLink: + name: "vcvt{neon_type[1].no}_{neon_type[0]}" + links: + - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" + arch: arm + - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" + arch: aarch64,arm64ec + + - name: "vqmovn_{neon_type[0]}" + doc: "Unsigned saturating extract narrow" + arguments: ["a: {neon_type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovn]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqxtn]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - [uint16x8_t, uint8x8_t] + - [uint32x4_t, uint16x4_t] + - [uint64x2_t, uint32x2_t] + compose: + - LLVMLink: + name: "vqmovn_{neon_type[1]}" + links: + - link: "llvm.arm.neon.vqmovnu.{neon_type[1]}" + arch: arm + - link: "llvm.aarch64.neon.uqxtn.{neon_type[1]}" + arch: aarch64,arm64ec + + - name: "vcle{neon_type.no}" + doc: "Compare unsigned less than or equal" + arguments: ["a: {neon_type}", "b: {neon_type}"] + return_type: "{neon_type}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type}"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhs]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - uint8x8_t + - uint8x16_t + - uint16x4_t + - uint16x8_t + - uint32x2_t + - uint32x4_t + compose: + - FnCall: [simd_le, [a, b]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld4]]}]] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x4_t, int8x8_t, '1'] + - ["*const i16", int16x4x4_t, int16x4_t, '2'] + - ["*const i32", int32x2x4_t, int32x2_t, '4'] + - ["*const i8", int8x16x4_t, int8x16_t, '1'] + - ["*const i16", int16x8x4_t, int16x8_t, '2'] + - ["*const i32", int32x4x4_t, int32x4_t, '4'] + - ["*const f32", float32x2x4_t, float32x2_t, '4'] + - ["*const f32", float32x4x4_t, float32x4_t, '4'] + compose: + - LLVMLink: + name: "vld4{neon_type[1].dup_nox}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld4dup.{neon_type[2]}.p0i8" + arch: arm + - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as *const i8', "{type[3]}"]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] + safety: + unsafe: [neon] + types: + - ["*const i8", int8x8x4_t, int8x8_t, 'p0i8'] + - ["*const i16", int16x4x4_t, int16x4_t, 'p0i16'] + - ["*const i32", int32x2x4_t, int32x2_t, 'p0i32'] + - ["*const i8", int8x16x4_t, int8x16_t, 'p0i8'] + - ["*const i16", int16x8x4_t, int16x8_t, 'p0i16'] + - ["*const i32", int32x4x4_t, int32x4_t, 'p0i32'] + - ["*const i64", int64x1x4_t, int64x1_t, 'p0i64'] + - ["*const f32", float32x2x4_t, float32x2_t, 'p0f32'] + - ["*const f32", float32x4x4_t, float32x4_t, 'p0f32'] + compose: + - LLVMLink: + name: "vld4{neon_type[1].dup_nox}" + arguments: + - "ptr: {type[0]}" + links: + - link: "llvm.aarch64.neon.ld4r.{neon_type[2]}.{type[3]}" + arch: aarch64,arm64ec + - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as _']] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg, ['target_arch = "arm"']] + - FnCall: [target_feature, ['enable = "neon,v7"']] + - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] + safety: + unsafe: [neon] + types: + - ["*const i64", int64x1x4_t] + compose: + - LLVMLink: + name: "vld4{neon_type[1].dup_nox}" + arguments: + - "ptr: *const i8" + - "size: i32" + links: + - link: "llvm.arm.neon.vld4dup.v1i64.p0i8" + arch: arm + - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as *const i8', '8']] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vld4]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*const u8", uint8x8x4_t, int8x8x4_t] + - ["*const u16", uint16x4x4_t, int16x4x4_t] + - ["*const u32", uint32x2x4_t, int32x2x4_t] + - ["*const u8", uint8x16x4_t, int8x16x4_t] + - ["*const u16", uint16x8x4_t, int16x8x4_t] + - ["*const u32", uint32x4x4_t, int32x4x4_t] + - ["*const p8", poly8x8x4_t, int8x8x4_t] + - ["*const p16", poly16x4x4_t, int16x4x4_t] + - ["*const p8", poly8x16x4_t, int8x16x4_t] + - ["*const p16", poly16x8x4_t, int16x8x4_t] + compose: + - FnCall: + - "transmute" + - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*const u64", uint64x1x4_t, int64x1x4_t] + compose: + - FnCall: + - "transmute" + - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] + + - name: "vld4{neon_type[1].dup_nox}" + doc: "Load single 4-element structure and replicate to all lanes of four registers" + arguments: ["a: {type[0]}"] + return_type: "{neon_type[1]}" + attr: + - FnCall: [target_feature, ['enable = "neon,aes"']] + - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] + - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] + - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] + - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] + safety: + unsafe: [neon] + types: + - ["*const p64", poly64x1x4_t, int64x1x4_t] + compose: + - FnCall: + - "transmute" + - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] + diff --git a/library/stdarch/crates/stdarch-gen-arm/src/assert_instr.rs b/library/stdarch/crates/stdarch-gen-arm/src/assert_instr.rs new file mode 100644 index 00000000000..ce1bbe8b55f --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/assert_instr.rs @@ -0,0 +1,372 @@ +use proc_macro2::TokenStream; +use quote::{format_ident, quote, ToTokens, TokenStreamExt}; +use serde::de::{self, MapAccess, Visitor}; +use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; +use std::fmt; + +use crate::{ + context::{self, Context}, + typekinds::{BaseType, BaseTypeKind}, + wildstring::WildString, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum InstructionAssertion { + Basic(WildString), + WithArgs(WildString, WildString), +} + +impl InstructionAssertion { + fn build(&mut self, ctx: &Context) -> context::Result { + match self { + InstructionAssertion::Basic(ws) => ws.build_acle(ctx.local), + InstructionAssertion::WithArgs(ws, args_ws) => [ws, args_ws] + .into_iter() + .try_for_each(|ws| ws.build_acle(ctx.local)), + } + } +} + +impl ToTokens for InstructionAssertion { + fn to_tokens(&self, tokens: &mut TokenStream) { + let instr = format_ident!( + "{}", + match self { + Self::Basic(instr) => instr, + Self::WithArgs(instr, _) => instr, + } + .to_string() + ); + tokens.append_all(quote! { #instr }); + + if let Self::WithArgs(_, args) = self { + let ex: TokenStream = args + .to_string() + .parse() + .expect("invalid instruction assertion arguments expression given"); + tokens.append_all(quote! {, #ex}) + } + } +} + +// Asserts that the given instruction is present for the intrinsic of the associated type bitsize. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(remote = "Self")] +pub struct InstructionAssertionMethodForBitsize { + pub default: InstructionAssertion, + pub byte: Option, + pub halfword: Option, + pub word: Option, + pub doubleword: Option, +} + +impl InstructionAssertionMethodForBitsize { + fn build(&mut self, ctx: &Context) -> context::Result { + if let Some(ref mut byte) = self.byte { + byte.build(ctx)? + } + if let Some(ref mut halfword) = self.halfword { + halfword.build(ctx)? + } + if let Some(ref mut word) = self.word { + word.build(ctx)? + } + if let Some(ref mut doubleword) = self.doubleword { + doubleword.build(ctx)? + } + self.default.build(ctx) + } +} + +impl Serialize for InstructionAssertionMethodForBitsize { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + InstructionAssertionMethodForBitsize { + default: InstructionAssertion::Basic(instr), + byte: None, + halfword: None, + word: None, + doubleword: None, + } => serializer.serialize_str(&instr.to_string()), + InstructionAssertionMethodForBitsize { + default: InstructionAssertion::WithArgs(instr, args), + byte: None, + halfword: None, + word: None, + doubleword: None, + } => { + let mut seq = serializer.serialize_seq(Some(2))?; + seq.serialize_element(&instr.to_string())?; + seq.serialize_element(&args.to_string())?; + seq.end() + } + _ => InstructionAssertionMethodForBitsize::serialize(self, serializer), + } + } +} + +impl<'de> Deserialize<'de> for InstructionAssertionMethodForBitsize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct IAMVisitor; + + impl<'de> Visitor<'de> for IAMVisitor { + type Value = InstructionAssertionMethodForBitsize; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("array, string or map") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(InstructionAssertionMethodForBitsize { + default: InstructionAssertion::Basic(value.parse().map_err(E::custom)?), + byte: None, + halfword: None, + word: None, + doubleword: None, + }) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + use serde::de::Error; + let make_err = + || Error::custom("invalid number of arguments passed to assert_instruction"); + let instruction = seq.next_element()?.ok_or_else(make_err)?; + let args = seq.next_element()?.ok_or_else(make_err)?; + + if let Some(true) = seq.size_hint().map(|len| len > 0) { + Err(make_err()) + } else { + Ok(InstructionAssertionMethodForBitsize { + default: InstructionAssertion::WithArgs(instruction, args), + byte: None, + halfword: None, + word: None, + doubleword: None, + }) + } + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + InstructionAssertionMethodForBitsize::deserialize( + de::value::MapAccessDeserializer::new(map), + ) + } + } + + deserializer.deserialize_any(IAMVisitor) + } +} + +/// Asserts that the given instruction is present for the intrinsic of the associated type. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(remote = "Self")] +pub struct InstructionAssertionMethod { + /// Instruction for integer intrinsics + pub default: InstructionAssertionMethodForBitsize, + /// Instruction for floating-point intrinsics (optional) + #[serde(default)] + pub float: Option, + /// Instruction for unsigned integer intrinsics (optional) + #[serde(default)] + pub unsigned: Option, +} + +impl InstructionAssertionMethod { + pub(crate) fn build(&mut self, ctx: &Context) -> context::Result { + if let Some(ref mut float) = self.float { + float.build(ctx)? + } + if let Some(ref mut unsigned) = self.unsigned { + unsigned.build(ctx)? + } + self.default.build(ctx) + } +} + +impl Serialize for InstructionAssertionMethod { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + InstructionAssertionMethod { + default: + InstructionAssertionMethodForBitsize { + default: InstructionAssertion::Basic(instr), + byte: None, + halfword: None, + word: None, + doubleword: None, + }, + float: None, + unsigned: None, + } => serializer.serialize_str(&instr.to_string()), + InstructionAssertionMethod { + default: + InstructionAssertionMethodForBitsize { + default: InstructionAssertion::WithArgs(instr, args), + byte: None, + halfword: None, + word: None, + doubleword: None, + }, + float: None, + unsigned: None, + } => { + let mut seq = serializer.serialize_seq(Some(2))?; + seq.serialize_element(&instr.to_string())?; + seq.serialize_element(&args.to_string())?; + seq.end() + } + _ => InstructionAssertionMethod::serialize(self, serializer), + } + } +} + +impl<'de> Deserialize<'de> for InstructionAssertionMethod { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct IAMVisitor; + + impl<'de> Visitor<'de> for IAMVisitor { + type Value = InstructionAssertionMethod; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("array, string or map") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + Ok(InstructionAssertionMethod { + default: InstructionAssertionMethodForBitsize { + default: InstructionAssertion::Basic(value.parse().map_err(E::custom)?), + byte: None, + halfword: None, + word: None, + doubleword: None, + }, + float: None, + unsigned: None, + }) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + use serde::de::Error; + let make_err = + || Error::custom("invalid number of arguments passed to assert_instruction"); + let instruction = seq.next_element()?.ok_or_else(make_err)?; + let args = seq.next_element()?.ok_or_else(make_err)?; + + if let Some(true) = seq.size_hint().map(|len| len > 0) { + Err(make_err()) + } else { + Ok(InstructionAssertionMethod { + default: InstructionAssertionMethodForBitsize { + default: InstructionAssertion::WithArgs(instruction, args), + byte: None, + halfword: None, + word: None, + doubleword: None, + }, + float: None, + unsigned: None, + }) + } + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + InstructionAssertionMethod::deserialize(de::value::MapAccessDeserializer::new(map)) + } + } + + deserializer.deserialize_any(IAMVisitor) + } +} + +#[derive(Debug)] +pub struct InstructionAssertionsForBaseType<'a>( + pub &'a Vec, + pub &'a Option<&'a BaseType>, +); + +impl<'a> ToTokens for InstructionAssertionsForBaseType<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.iter().for_each( + |InstructionAssertionMethod { + default, + float, + unsigned, + }| { + let kind = self.1.map(|ty| ty.kind()); + let instruction = match (kind, float, unsigned) { + (None, float, unsigned) if float.is_some() || unsigned.is_some() => { + unreachable!( + "cannot determine the base type kind for instruction assertion: {self:#?}") + } + (Some(BaseTypeKind::Float), Some(float), _) => float, + (Some(BaseTypeKind::UInt), _, Some(unsigned)) => unsigned, + _ => default, + }; + + let bitsize = self.1.and_then(|ty| ty.get_size().ok()); + let instruction = match (bitsize, instruction) { + ( + Some(8), + InstructionAssertionMethodForBitsize { + byte: Some(byte), .. + }, + ) => byte, + ( + Some(16), + InstructionAssertionMethodForBitsize { + halfword: Some(halfword), + .. + }, + ) => halfword, + ( + Some(32), + InstructionAssertionMethodForBitsize { + word: Some(word), .. + }, + ) => word, + ( + Some(64), + InstructionAssertionMethodForBitsize { + doubleword: Some(doubleword), + .. + }, + ) => doubleword, + (_, InstructionAssertionMethodForBitsize { default, .. }) => default, + }; + + tokens.append_all(quote! { #[cfg_attr(test, assert_instr(#instruction))]}) + }, + ); + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/context.rs b/library/stdarch/crates/stdarch-gen-arm/src/context.rs new file mode 100644 index 00000000000..8405428b7ad --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/context.rs @@ -0,0 +1,263 @@ +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, usize}; + +use crate::{ + expression::Expression, + input::{InputSet, InputType}, + intrinsic::{Constraint, Intrinsic, Signature}, + matching::SizeMatchable, + predicate_forms::PredicateForm, + typekinds::{ToRepr, TypeKind}, + wildcards::Wildcard, + wildstring::WildString, +}; + +/// Maximum SVE vector size +const SVE_VECTOR_MAX_SIZE: u32 = 2048; +/// Vector register size +const VECTOR_REG_SIZE: u32 = 128; + +/// Generator result +pub type Result = std::result::Result; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArchitectureSettings { + #[serde(alias = "arch")] + pub arch_name: String, + pub target_feature: Vec, + #[serde(alias = "llvm_prefix")] + pub llvm_link_prefix: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GlobalContext { + pub arch_cfgs: Vec, + #[serde(default)] + pub uses_neon_types: bool, +} + +/// Context of an intrinsic group +#[derive(Debug, Clone, Default)] +pub struct GroupContext { + /// LLVM links to target input sets + pub links: HashMap, +} + +#[derive(Debug, Clone, Copy)] +pub enum VariableType { + Argument, + Internal, +} + +#[derive(Debug, Clone)] +pub struct LocalContext { + pub signature: Signature, + + pub input: InputSet, + + pub substitutions: HashMap, + pub variables: HashMap, +} + +impl LocalContext { + pub fn new(input: InputSet, original: &Intrinsic) -> LocalContext { + LocalContext { + signature: original.signature.clone(), + input, + substitutions: HashMap::new(), + variables: HashMap::new(), + } + } + + pub fn provide_type_wildcard(&self, wildcard: &Wildcard) -> Result { + let err = || { + format!( + "provide_type_wildcard() wildcard {{{wildcard}}} not found for {}", + &self.signature.name.to_string() + ) + }; + + /* If the type is already a vector then we can just return the vector */ + let make_neon = |tuple_size| { + move |ty| match ty { + TypeKind::Vector(_) => Ok(ty), + _ => TypeKind::make_vector(ty, false, tuple_size), + } + }; + let make_sve = |tuple_size| move |ty| TypeKind::make_vector(ty, true, tuple_size); + + match wildcard { + Wildcard::Type(idx) => self.input.typekind(*idx).ok_or_else(err), + Wildcard::NEONType(idx, tuple_size, _) => self + .input + .typekind(*idx) + .ok_or_else(|| { + dbg!("{:?}", &self); + err() + }) + .and_then(make_neon(*tuple_size)), + Wildcard::SVEType(idx, tuple_size) => self + .input + .typekind(*idx) + .ok_or_else(err) + .and_then(make_sve(*tuple_size)), + Wildcard::Predicate(idx) => self.input.typekind(*idx).map_or_else( + || { + if idx.is_none() && self.input.types_len() == 1 { + Err(err()) + } else { + Err(format!( + "there is no type at index {} to infer the predicate from", + idx.unwrap_or(0) + )) + } + }, + |ref ty| TypeKind::make_predicate_from(ty), + ), + Wildcard::MaxPredicate => self + .input + .iter() + .filter_map(|arg| arg.typekind()) + .max_by(|x, y| { + x.base_type() + .and_then(|bt| bt.get_size().ok()) + .unwrap_or(0) + .cmp(&y.base_type().and_then(|bt| bt.get_size().ok()).unwrap_or(0)) + }) + .map_or_else( + || Err("there are no types available to infer the predicate from".to_string()), + TypeKind::make_predicate_from, + ), + Wildcard::Scale(w, as_ty) => { + let mut ty = self.provide_type_wildcard(w)?; + if let Some(vty) = ty.vector_mut() { + let base_ty = if let Some(w) = as_ty.wildcard() { + *self.provide_type_wildcard(w)?.base_type().unwrap() + } else { + *as_ty.base_type().unwrap() + }; + vty.cast_base_type_as(base_ty) + } + Ok(ty) + } + _ => Err(err()), + } + } + + pub fn provide_substitution_wildcard(&self, wildcard: &Wildcard) -> Result { + let err = || Err(format!("wildcard {{{wildcard}}} not found")); + + match wildcard { + Wildcard::SizeLiteral(idx) => self.input.typekind(*idx) + .map_or_else(err, |ty| Ok(ty.size_literal())), + Wildcard::Size(idx) => self.input.typekind(*idx) + .map_or_else(err, |ty| Ok(ty.size())), + Wildcard::SizeMinusOne(idx) => self.input.typekind(*idx) + .map_or_else(err, |ty| Ok((ty.size().parse::().unwrap()-1).to_string())), + Wildcard::SizeInBytesLog2(idx) => self.input.typekind(*idx) + .map_or_else(err, |ty| Ok(ty.size_in_bytes_log2())), + Wildcard::NVariant if self.substitutions.get(wildcard).is_none() => Ok(String::new()), + Wildcard::TypeKind(idx, opts) => { + self.input.typekind(*idx) + .map_or_else(err, |ty| { + let literal = if let Some(opts) = opts { + opts.contains(ty.base_type().map(|bt| *bt.kind()).ok_or_else(|| { + format!("cannot retrieve a type literal out of {ty}") + })?) + .then(|| ty.type_kind()) + .unwrap_or_default() + } else { + ty.type_kind() + }; + Ok(literal) + }) + } + Wildcard::PredicateForms(_) => self + .input + .iter() + .find_map(|arg| { + if let InputType::PredicateForm(pf) = arg { + Some(pf.get_suffix().to_string()) + } else { + None + } + }) + .ok_or_else(|| unreachable!("attempting to render a predicate form wildcard, but no predicate form was compiled for it")), + _ => self + .substitutions + .get(wildcard) + .map_or_else(err, |s| Ok(s.clone())), + } + } + + pub fn make_assertion_from_constraint(&self, constraint: &Constraint) -> Result { + match constraint { + Constraint::AnyI32 { + variable, + any_values, + } => { + let where_ex = any_values + .iter() + .map(|value| format!("{variable} == {value}")) + .join(" || "); + Ok(Expression::MacroCall("static_assert".to_string(), where_ex)) + } + Constraint::RangeI32 { + variable, + range: SizeMatchable::Matched(range), + } => Ok(Expression::MacroCall( + "static_assert_range".to_string(), + format!( + "{variable}, {min}, {max}", + min = range.start(), + max = range.end() + ), + )), + Constraint::SVEMaxElems { + variable, + sve_max_elems_type: ty, + } + | Constraint::VecMaxElems { + variable, + vec_max_elems_type: ty, + } => { + if !self.input.is_empty() { + let higher_limit = match constraint { + Constraint::SVEMaxElems { .. } => SVE_VECTOR_MAX_SIZE, + Constraint::VecMaxElems { .. } => VECTOR_REG_SIZE, + _ => unreachable!(), + }; + + let max = ty.base_type() + .map(|ty| ty.get_size()) + .transpose()? + .map_or_else( + || Err(format!("can't make an assertion out of constraint {self:?}: no valid type is present")), + |bitsize| Ok(higher_limit / bitsize - 1))?; + Ok(Expression::MacroCall( + "static_assert_range".to_string(), + format!("{variable}, 0, {max}"), + )) + } else { + Err(format!("can't make an assertion out of constraint {self:?}: no types are being used")) + } + } + _ => unreachable!("constraints were not built successfully!"), + } + } + + pub fn predicate_form(&self) -> Option<&PredicateForm> { + self.input.iter().find_map(|arg| arg.predicate_form()) + } + + pub fn n_variant_op(&self) -> Option<&WildString> { + self.input.iter().find_map(|arg| arg.n_variant_op()) + } +} + +pub struct Context<'ctx> { + pub local: &'ctx mut LocalContext, + pub group: &'ctx mut GroupContext, + pub global: &'ctx GlobalContext, +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/expression.rs b/library/stdarch/crates/stdarch-gen-arm/src/expression.rs new file mode 100644 index 00000000000..83984679588 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/expression.rs @@ -0,0 +1,576 @@ +use itertools::Itertools; +use lazy_static::lazy_static; +use proc_macro2::{Literal, Punct, Spacing, TokenStream}; +use quote::{format_ident, quote, ToTokens, TokenStreamExt}; +use regex::Regex; +use serde::de::{self, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; +use std::fmt; +use std::str::FromStr; + +use crate::intrinsic::Intrinsic; +use crate::{ + context::{self, Context, VariableType}, + intrinsic::{Argument, LLVMLink, StaticDefinition}, + matching::{MatchKindValues, MatchSizeValues}, + typekinds::{BaseType, BaseTypeKind, TypeKind}, + wildcards::Wildcard, + wildstring::WildString, +}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum IdentifierType { + Variable, + Symbol, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum LetVariant { + Basic(WildString, Box), + WithType(WildString, TypeKind, Box), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FnCall( + /// Function pointer + pub Box, + /// Function arguments + pub Vec, + /// Function turbofish arguments + #[serde(default)] + pub Vec, +); + +impl FnCall { + pub fn new_expression(fn_ptr: Expression, arguments: Vec) -> Expression { + FnCall(Box::new(fn_ptr), arguments, Vec::new()).into() + } + + pub fn is_llvm_link_call(&self, llvm_link_name: &String) -> bool { + self.is_expected_call(llvm_link_name) + } + + pub fn is_target_feature_call(&self) -> bool { + self.is_expected_call("target_feature") + } + + pub fn is_expected_call(&self, fn_call_name: &str) -> bool { + if let Expression::Identifier(fn_name, IdentifierType::Symbol) = self.0.as_ref() { + &fn_name.to_string() == fn_call_name + } else { + false + } + } + + pub fn pre_build(&mut self, ctx: &mut Context) -> context::Result { + self.0.pre_build(ctx)?; + self.1 + .iter_mut() + .chain(self.2.iter_mut()) + .try_for_each(|ex| ex.pre_build(ctx)) + } + + pub fn build(&mut self, intrinsic: &Intrinsic, ctx: &mut Context) -> context::Result { + self.0.build(intrinsic, ctx)?; + self.1 + .iter_mut() + .chain(self.2.iter_mut()) + .try_for_each(|ex| ex.build(intrinsic, ctx)) + } +} + +impl ToTokens for FnCall { + fn to_tokens(&self, tokens: &mut TokenStream) { + let FnCall(fn_ptr, arguments, turbofish) = self; + + fn_ptr.to_tokens(tokens); + + if !turbofish.is_empty() { + tokens.append_all(quote! {::<#(#turbofish),*>}); + } + + tokens.append_all(quote! { (#(#arguments),*) }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(remote = "Self", deny_unknown_fields)] +pub enum Expression { + /// (Re)Defines a variable + Let(LetVariant), + /// Performs a variable assignment operation + Assign(String, Box), + /// Performs a macro call + MacroCall(String, String), + /// Performs a function call + FnCall(FnCall), + /// Performs a method call. The following: + /// `MethodCall: ["$object", "to_string", []]` + /// is tokenized as: + /// `object.to_string()`. + MethodCall(Box, String, Vec), + /// Symbol identifier name, prepend with a `$` to treat it as a scope variable + /// which engages variable tracking and enables inference. + /// E.g. `my_function_name` for a generic symbol or `$my_variable` for + /// a variable. + Identifier(WildString, IdentifierType), + /// Constant signed integer number expression + IntConstant(i32), + /// Constant floating point number expression + FloatConstant(f32), + /// Constant boolean expression, either `true` or `false` + BoolConstant(bool), + /// Array expression + Array(Vec), + + // complex expressions + /// Makes an LLVM link. + /// + /// It stores the link's function name in the wildcard `{llvm_link}`, for use in + /// subsequent expressions. + LLVMLink(LLVMLink), + /// Casts the given expression to the specified (unchecked) type + CastAs(Box, String), + /// Returns the LLVM `undef` symbol + SvUndef, + /// Multiplication + Multiply(Box, Box), + /// Xor + Xor(Box, Box), + /// Converts the specified constant to the specified type's kind + ConvertConst(TypeKind, i32), + /// Yields the given type in the Rust representation + Type(TypeKind), + + MatchSize(TypeKind, MatchSizeValues>), + MatchKind(TypeKind, MatchKindValues>), +} + +impl Expression { + pub fn pre_build(&mut self, ctx: &mut Context) -> context::Result { + match self { + Self::FnCall(fn_call) => fn_call.pre_build(ctx), + Self::MethodCall(cl_ptr_ex, _, arg_exs) => { + cl_ptr_ex.pre_build(ctx)?; + arg_exs.iter_mut().try_for_each(|ex| ex.pre_build(ctx)) + } + Self::Let(LetVariant::Basic(_, ex) | LetVariant::WithType(_, _, ex)) => { + ex.pre_build(ctx) + } + Self::CastAs(ex, _) => ex.pre_build(ctx), + Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { + lhs.pre_build(ctx)?; + rhs.pre_build(ctx) + } + Self::MatchSize(match_ty, values) => { + *self = *values.get(match_ty, ctx.local)?.to_owned(); + self.pre_build(ctx) + } + Self::MatchKind(match_ty, values) => { + *self = *values.get(match_ty, ctx.local)?.to_owned(); + self.pre_build(ctx) + } + _ => Ok(()), + } + } + + pub fn build(&mut self, intrinsic: &Intrinsic, ctx: &mut Context) -> context::Result { + match self { + Self::LLVMLink(link) => link.build_and_save(ctx), + Self::Identifier(identifier, id_type) => { + identifier.build_acle(ctx.local)?; + + if let IdentifierType::Variable = id_type { + ctx.local + .variables + .get(&identifier.to_string()) + .map(|_| ()) + .ok_or_else(|| format!("invalid variable {identifier} being referenced")) + } else { + Ok(()) + } + } + Self::FnCall(fn_call) => { + fn_call.build(intrinsic, ctx)?; + + if let Some(llvm_link_name) = ctx.local.substitutions.get(&Wildcard::LLVMLink) { + if fn_call.is_llvm_link_call(llvm_link_name) { + *self = intrinsic + .llvm_link() + .expect("got LLVMLink wildcard without a LLVM link in `compose`") + .apply_conversions_to_call(fn_call.clone(), ctx.local)? + } + } + + Ok(()) + } + Self::MethodCall(cl_ptr_ex, _, arg_exs) => { + cl_ptr_ex.build(intrinsic, ctx)?; + arg_exs + .iter_mut() + .try_for_each(|ex| ex.build(intrinsic, ctx)) + } + Self::Let(variant) => { + let (var_name, ex, ty) = match variant { + LetVariant::Basic(var_name, ex) => (var_name, ex, None), + LetVariant::WithType(var_name, ty, ex) => { + if let Some(w) = ty.wildcard() { + ty.populate_wildcard(ctx.local.provide_type_wildcard(w)?)?; + } + (var_name, ex, Some(ty.to_owned())) + } + }; + + var_name.build_acle(ctx.local)?; + ctx.local.variables.insert( + var_name.to_string(), + ( + ty.unwrap_or_else(|| TypeKind::Custom("unknown".to_string())), + VariableType::Internal, + ), + ); + ex.build(intrinsic, ctx) + } + Self::CastAs(ex, _) => ex.build(intrinsic, ctx), + Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { + lhs.build(intrinsic, ctx)?; + rhs.build(intrinsic, ctx) + } + Self::ConvertConst(ty, num) => { + if let Some(w) = ty.wildcard() { + *ty = ctx.local.provide_type_wildcard(w)? + } + + if let Some(BaseType::Sized(BaseTypeKind::Float, _)) = ty.base() { + *self = Expression::FloatConstant(*num as f32) + } else { + *self = Expression::IntConstant(*num) + } + Ok(()) + } + Self::Type(ty) => { + if let Some(w) = ty.wildcard() { + *ty = ctx.local.provide_type_wildcard(w)? + } + + Ok(()) + } + _ => Ok(()), + } + } + + /// True if the expression requires an `unsafe` context in a safe function. + /// + /// The classification is somewhat fuzzy, based on actual usage (e.g. empirical function names) + /// rather than a full parse. This is a reasonable approach because mistakes here will usually + /// be caught at build time: + /// + /// - Missing an `unsafe` is a build error. + /// - An unnecessary `unsafe` is a warning, made into an error by the CI's `-D warnings`. + /// + /// This **panics** if it encounters an expression that shouldn't appear in a safe function at + /// all (such as `SvUndef`). + pub fn requires_unsafe_wrapper(&self, ctx_fn: &str) -> bool { + match self { + // The call will need to be unsafe, but the declaration does not. + Self::LLVMLink(..) => false, + // Identifiers, literals and type names are never unsafe. + Self::Identifier(..) => false, + Self::IntConstant(..) => false, + Self::FloatConstant(..) => false, + Self::BoolConstant(..) => false, + Self::Type(..) => false, + Self::ConvertConst(..) => false, + // Nested structures that aren't inherently unsafe, but could contain other expressions + // that might be. + Self::Assign(_var, exp) => exp.requires_unsafe_wrapper(ctx_fn), + Self::Let(LetVariant::Basic(_, exp) | LetVariant::WithType(_, _, exp)) => { + exp.requires_unsafe_wrapper(ctx_fn) + } + Self::Array(exps) => exps.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)), + Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { + lhs.requires_unsafe_wrapper(ctx_fn) || rhs.requires_unsafe_wrapper(ctx_fn) + } + Self::CastAs(exp, _ty) => exp.requires_unsafe_wrapper(ctx_fn), + // Functions and macros can be unsafe, but can also contain other expressions. + Self::FnCall(FnCall(fn_exp, args, turbo_args)) => { + let fn_name = fn_exp.to_string(); + fn_exp.requires_unsafe_wrapper(ctx_fn) + || fn_name.starts_with("_sv") + || fn_name.starts_with("simd_") + || fn_name.ends_with("transmute") + || args.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) + || turbo_args + .iter() + .any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) + } + Self::MethodCall(exp, fn_name, args) => match fn_name.as_str() { + // `as_signed` and `as_unsigned` are unsafe because they're trait methods with + // target features to allow use on feature-dependent types (such as SVE vectors). + // We can safely wrap them here. + "as_signed" => true, + "as_unsigned" => true, + _ => { + exp.requires_unsafe_wrapper(ctx_fn) + || args.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) + } + }, + // We only use macros to check const generics (using static assertions). + Self::MacroCall(_name, _args) => false, + // Materialising uninitialised values is always unsafe, and we avoid it in safe + // functions. + Self::SvUndef => panic!("Refusing to wrap unsafe SvUndef in safe function '{ctx_fn}'."), + // Variants that aren't tokenised. We shouldn't encounter these here. + Self::MatchKind(..) => { + unimplemented!("The unsafety of {self:?} cannot be determined in '{ctx_fn}'.") + } + Self::MatchSize(..) => { + unimplemented!("The unsafety of {self:?} cannot be determined in '{ctx_fn}'.") + } + } + } +} + +impl FromStr for Expression { + type Err = String; + + fn from_str(s: &str) -> Result { + lazy_static! { + static ref MACRO_RE: Regex = + Regex::new(r"^(?P[\w\d_]+)!\((?P.*?)\);?$").unwrap(); + } + + if s == "SvUndef" { + Ok(Expression::SvUndef) + } else if MACRO_RE.is_match(s) { + let c = MACRO_RE.captures(s).unwrap(); + let ex = c["ex"].to_string(); + let _: TokenStream = ex + .parse() + .map_err(|e| format!("could not parse macro call expression: {e:#?}"))?; + Ok(Expression::MacroCall(c["name"].to_string(), ex)) + } else { + let (s, id_type) = if let Some(varname) = s.strip_prefix('$') { + (varname, IdentifierType::Variable) + } else { + (s, IdentifierType::Symbol) + }; + let identifier = s.trim().parse()?; + Ok(Expression::Identifier(identifier, id_type)) + } + } +} + +impl From for Expression { + fn from(fn_call: FnCall) -> Self { + Expression::FnCall(fn_call) + } +} + +impl From for Expression { + fn from(ws: WildString) -> Self { + Expression::Identifier(ws, IdentifierType::Symbol) + } +} + +impl From<&Argument> for Expression { + fn from(a: &Argument) -> Self { + Expression::Identifier(a.name.to_owned(), IdentifierType::Variable) + } +} + +impl TryFrom<&StaticDefinition> for Expression { + type Error = String; + + fn try_from(sd: &StaticDefinition) -> Result { + match sd { + StaticDefinition::Constant(imm) => Ok(imm.into()), + StaticDefinition::Generic(t) => t.parse(), + } + } +} + +impl fmt::Display for Expression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Identifier(identifier, kind) => { + write!( + f, + "{}{identifier}", + matches!(kind, IdentifierType::Variable) + .then_some("$") + .unwrap_or_default() + ) + } + Self::MacroCall(name, expression) => { + write!(f, "{name}!({expression})") + } + _ => Err(fmt::Error), + } + } +} + +impl ToTokens for Expression { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::Let(LetVariant::Basic(var_name, exp)) => { + let var_ident = format_ident!("{}", var_name.to_string()); + tokens.append_all(quote! { let #var_ident = #exp }) + } + Self::Let(LetVariant::WithType(var_name, ty, exp)) => { + let var_ident = format_ident!("{}", var_name.to_string()); + tokens.append_all(quote! { let #var_ident: #ty = #exp }) + } + Self::Assign(var_name, exp) => { + /* If we are dereferencing a variable to assign a value \ + * the 'format_ident!' macro does not like the asterix */ + let var_name_str: &str; + + if let Some(ch) = var_name.chars().nth(0) { + /* Manually append the asterix and split out the rest of + * the variable name */ + if ch == '*' { + tokens.append(Punct::new('*', Spacing::Alone)); + var_name_str = &var_name[1..var_name.len()]; + } else { + var_name_str = var_name.as_str(); + } + } else { + /* Should not be reached as you cannot have a variable + * without a name */ + panic!("Invalid variable name, must be at least one character") + } + + let var_ident = format_ident!("{}", var_name_str); + tokens.append_all(quote! { #var_ident = #exp }) + } + Self::MacroCall(name, ex) => { + let name = format_ident!("{name}"); + let ex: TokenStream = ex.parse().unwrap(); + tokens.append_all(quote! { #name!(#ex) }) + } + Self::FnCall(fn_call) => fn_call.to_tokens(tokens), + Self::MethodCall(exp, fn_name, args) => { + let fn_ident = format_ident!("{}", fn_name); + tokens.append_all(quote! { #exp.#fn_ident(#(#args),*) }) + } + Self::Identifier(identifier, _) => { + assert!( + !identifier.has_wildcards(), + "expression {self:#?} was not built before calling to_tokens" + ); + identifier + .to_string() + .parse::() + .expect(format!("invalid syntax: {:?}", self).as_str()) + .to_tokens(tokens); + } + Self::IntConstant(n) => tokens.append(Literal::i32_unsuffixed(*n)), + Self::FloatConstant(n) => tokens.append(Literal::f32_unsuffixed(*n)), + Self::BoolConstant(true) => tokens.append(format_ident!("true")), + Self::BoolConstant(false) => tokens.append(format_ident!("false")), + Self::Array(vec) => tokens.append_all(quote! { [ #(#vec),* ] }), + Self::LLVMLink(link) => link.to_tokens(tokens), + Self::CastAs(ex, ty) => { + let ty: TokenStream = ty.parse().expect("invalid syntax"); + tokens.append_all(quote! { #ex as #ty }) + } + Self::SvUndef => tokens.append_all(quote! { simd_reinterpret(()) }), + Self::Multiply(lhs, rhs) => tokens.append_all(quote! { #lhs * #rhs }), + Self::Xor(lhs, rhs) => tokens.append_all(quote! { #lhs ^ #rhs }), + Self::Type(ty) => ty.to_tokens(tokens), + _ => unreachable!("{self:?} cannot be converted to tokens."), + } + } +} + +impl Serialize for Expression { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::IntConstant(v) => serializer.serialize_i32(*v), + Self::FloatConstant(v) => serializer.serialize_f32(*v), + Self::BoolConstant(v) => serializer.serialize_bool(*v), + Self::Identifier(..) => serializer.serialize_str(&self.to_string()), + Self::MacroCall(..) => serializer.serialize_str(&self.to_string()), + _ => Expression::serialize(self, serializer), + } + } +} + +impl<'de> Deserialize<'de> for Expression { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct CustomExpressionVisitor; + + impl<'de> Visitor<'de> for CustomExpressionVisitor { + type Value = Expression; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("integer, float, boolean, string or map") + } + + fn visit_bool(self, v: bool) -> Result + where + E: de::Error, + { + Ok(Expression::BoolConstant(v)) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + Ok(Expression::IntConstant(v as i32)) + } + + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + Ok(Expression::IntConstant(v as i32)) + } + + fn visit_f64(self, v: f64) -> Result + where + E: de::Error, + { + Ok(Expression::FloatConstant(v as f32)) + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + FromStr::from_str(value).map_err(de::Error::custom) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + let arr = std::iter::from_fn(|| seq.next_element::().transpose()) + .try_collect()?; + Ok(Expression::Array(arr)) + } + + fn visit_map(self, map: M) -> Result + where + M: MapAccess<'de>, + { + // `MapAccessDeserializer` is a wrapper that turns a `MapAccess` + // into a `Deserializer`, allowing it to be used as the input to T's + // `Deserialize` implementation. T then deserializes itself using + // the entries from the map visitor. + Expression::deserialize(de::value::MapAccessDeserializer::new(map)) + } + } + + deserializer.deserialize_any(CustomExpressionVisitor) + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/fn_suffix.rs b/library/stdarch/crates/stdarch-gen-arm/src/fn_suffix.rs new file mode 100644 index 00000000000..9f7827776e0 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/fn_suffix.rs @@ -0,0 +1,308 @@ +use std::fmt::{self}; + +/* This file is acting as a bridge between the old neon types and how they + * have a fairly complex way of picking suffixes and the new world. If possible + * it would be good to clean this up. At least it is self contained and the + * logic simple */ +use crate::typekinds::{BaseType, BaseTypeKind, TypeKind, VectorType}; +use serde::{Deserialize, Serialize}; + +use std::str::FromStr; + +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Deserialize, Serialize)] +pub enum SuffixKind { + Normal, + Base, + NoQ, + NSuffix, + NoQNSuffix, + DupNox, + Dup, + /* Get the number of lanes or panic if there are not any Lanes */ + Lane, + Rot270, + Rot270Lane, + Rot270LaneQ, + Rot180, + Rot180Lane, + Rot180LaneQ, + Rot90, + Rot90Lane, + Rot90LaneQ, + /* Force the type to be unsigned */ + Unsigned, + Tuple, + NoX, + BaseByteSize, + LaneNoX, + LaneQNoX, +} + +pub fn type_to_size(str_type: &str) -> i32 { + match str_type { + "int8x8_t" | "int8x16_t" | "i8" | "s8" | "uint8x8_t" | "uint8x16_t" | "u8" + | "poly8x8_t" | "poly8x16_t" => 8, + "int16x4_t" | "int16x8_t" | "i16" | "s16" | "uint16x4_t" | "uint16x8_t" | "u16" + | "float16x4_t" | "float16x8_t" | "_f16" | "poly16x4_t" | "poly16x8_t" => 16, + "int32x2_t" | "int32x4_t" | "i32" | "s32" | "uint32x2_t" | "uint32x4_t" | "u32" + | "float32x2_t" | "float32x4_t" | "f32" => 32, + "int64x1_t" | "int64x2_t" | "i64" | "s64" | "uint64x1_t" | "uint64x2_t" | "u64" + | "float64x1_t" | "float64x2_t" | "f64" | "poly64x1_t" | "poly64x2_t" | "p64" => 64, + "p128" => 128, + _ => panic!("unknown type: {str_type}"), + } +} + +fn neon_get_base_and_char(ty: &VectorType) -> (u32, char, bool) { + let lanes = ty.lanes(); + match ty.base_type() { + BaseType::Sized(BaseTypeKind::Float, size) => (*size, 'f', *size * lanes == 128), + BaseType::Sized(BaseTypeKind::Int, size) => (*size, 's', *size * lanes == 128), + BaseType::Sized(BaseTypeKind::UInt, size) => (*size, 'u', *size * lanes == 128), + BaseType::Sized(BaseTypeKind::Poly, size) => (*size, 'p', *size * lanes == 128), + _ => panic!("Unhandled {:?}", ty), + } +} + +/* @TODO + * for the chained enum types we can safely delete them as we can index the + * types array */ +pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String { + match type_kind { + TypeKind::Vector(ty) => { + let tuple_size = ty.tuple_size().map_or(0, |t| t.to_int()); + let (base_size, prefix_char, requires_q) = neon_get_base_and_char(&ty); + let lanes = ty.lanes(); + match suffix_kind { + SuffixKind::Normal => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + if tuple_size > 0 { + str_suffix.push_str("_x"); + str_suffix.push_str(tuple_size.to_string().as_str()); + } + return str_suffix; + } + SuffixKind::NSuffix => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push_str("_n_"); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + return str_suffix; + } + + SuffixKind::NoQ => format!("_{}{}", prefix_char, base_size), + SuffixKind::NoQNSuffix => format!("_n{}{}", prefix_char, base_size), + + SuffixKind::Unsigned => { + let t = type_kind.to_string(); + if t.starts_with("u") { + return t; + } + return format!("u{}", t); + } + SuffixKind::Lane => { + if lanes == 0 { + panic!("type {} has no lanes!", type_kind.to_string()) + } else { + format!("{}", lanes) + } + } + SuffixKind::Tuple => { + if tuple_size == 0 { + panic!("type {} has no lanes!", type_kind.to_string()) + } else { + format!("{}", tuple_size) + } + } + SuffixKind::Base => base_size.to_string(), + SuffixKind::NoX => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + return str_suffix; + } + SuffixKind::Dup => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push_str("dup_"); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + if tuple_size > 0 { + str_suffix.push_str("_x"); + str_suffix.push_str(tuple_size.to_string().as_str()); + } + return str_suffix; + } + SuffixKind::DupNox => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push_str("dup_"); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + return str_suffix; + } + SuffixKind::LaneNoX => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push_str("lane_"); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + return str_suffix; + } + SuffixKind::LaneQNoX => { + let mut str_suffix: String = String::new(); + if requires_q { + str_suffix.push('q'); + } + str_suffix.push('_'); + str_suffix.push_str("laneq_"); + str_suffix.push(prefix_char); + str_suffix.push_str(base_size.to_string().as_str()); + return str_suffix; + } + SuffixKind::Rot270 => { + if requires_q { + return format!("q_rot270_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot270_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot270Lane => { + if requires_q { + return format!("q_rot270_lane_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot270_lane_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot270LaneQ => { + if requires_q { + return format!("q_rot270_laneq_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot270_laneq_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot180 => { + if requires_q { + return format!("q_rot180_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot180_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot180Lane => { + if requires_q { + return format!("q_rot180_lane_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot180_lane_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot180LaneQ => { + if requires_q { + return format!("q_rot180_laneq_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot180_laneq_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot90 => { + if requires_q { + return format!("q_rot90_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot90_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot90Lane => { + if requires_q { + return format!("q_rot90_lane_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot90_lane_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::Rot90LaneQ => { + if requires_q { + return format!("q_rot90_laneq_{}{}", prefix_char, base_size.to_string()); + } + return format!("_rot90_laneq_{}{}", prefix_char, base_size.to_string()); + } + SuffixKind::BaseByteSize => format!("{}", base_size / 8), + } + } + _ => panic!("Cannot only make neon vector types suffixed"), + } +} + +impl FromStr for SuffixKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "no" => Ok(SuffixKind::Normal), + "noq" => Ok(SuffixKind::NoQ), + "N" => Ok(SuffixKind::NSuffix), + "noq_N" => Ok(SuffixKind::NoQNSuffix), + "dup_nox" => Ok(SuffixKind::DupNox), + "dup" => Ok(SuffixKind::Dup), + "lane" => Ok(SuffixKind::Lane), + "base" => Ok(SuffixKind::Base), + "tuple" => Ok(SuffixKind::Tuple), + "rot270" => Ok(SuffixKind::Rot270), + "rot270_lane" => Ok(SuffixKind::Rot270Lane), + "rot270_laneq" => Ok(SuffixKind::Rot270LaneQ), + "rot90" => Ok(SuffixKind::Rot90), + "rot90_lane" => Ok(SuffixKind::Rot90Lane), + "rot90_laneq" => Ok(SuffixKind::Rot90LaneQ), + "rot180" => Ok(SuffixKind::Rot180), + "rot180_lane" => Ok(SuffixKind::Rot180LaneQ), + "rot180_laneq" => Ok(SuffixKind::Rot180LaneQ), + "u" => Ok(SuffixKind::Unsigned), + "nox" => Ok(SuffixKind::NoX), + "base_byte_size" => Ok(SuffixKind::BaseByteSize), + "lane_nox" => Ok(SuffixKind::LaneNoX), + "laneq_nox" => Ok(SuffixKind::LaneQNoX), + _ => Err(format!("unknown suffix type: {}", s)), + } + } +} + +impl fmt::Display for SuffixKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SuffixKind::Normal => write!(f, "normal"), + SuffixKind::NoQ => write!(f, "NoQ"), + SuffixKind::NSuffix => write!(f, "NSuffix"), + SuffixKind::NoQNSuffix => write!(f, "NoQNSuffix"), + SuffixKind::DupNox => write!(f, "DupNox"), + SuffixKind::Dup => write!(f, "Dup",), + SuffixKind::Lane => write!(f, "Lane"), + SuffixKind::LaneNoX => write!(f, "LaneNoX"), + SuffixKind::LaneQNoX => write!(f, "LaneQNoX"), + SuffixKind::Base => write!(f, "Base"), + SuffixKind::Rot270 => write!(f, "Rot270",), + SuffixKind::Rot270Lane => write!(f, "Rot270Lane"), + SuffixKind::Rot270LaneQ => write!(f, "Rot270LaneQ"), + SuffixKind::Rot90 => write!(f, "Rot90",), + SuffixKind::Rot90Lane => write!(f, "Rot90Lane"), + SuffixKind::Rot90LaneQ => write!(f, "Rot90LaneQ"), + SuffixKind::Rot180 => write!(f, "Rot180",), + SuffixKind::Rot180Lane => write!(f, "Rot180Lane"), + SuffixKind::Rot180LaneQ => write!(f, "Rot180LaneQ"), + SuffixKind::Unsigned => write!(f, "Unsigned"), + SuffixKind::Tuple => write!(f, "Tuple"), + SuffixKind::NoX => write!(f, "NoX"), + SuffixKind::BaseByteSize => write!(f, "BaseByteSize"), + } + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/input.rs b/library/stdarch/crates/stdarch-gen-arm/src/input.rs new file mode 100644 index 00000000000..bb2414adec2 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/input.rs @@ -0,0 +1,432 @@ +use itertools::Itertools; +use serde::{de, Deserialize, Deserializer, Serialize}; + +use crate::{ + context::{self, GlobalContext}, + intrinsic::Intrinsic, + predicate_forms::{PredicateForm, PredicationMask, PredicationMethods}, + typekinds::TypeKind, + wildstring::WildString, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum InputType { + /// PredicateForm variant argument + #[serde(skip)] // Predicate forms have their own dedicated deserialization field. Skip. + PredicateForm(PredicateForm), + /// Operand from which to generate an N variant + #[serde(skip)] + NVariantOp(Option), + /// TypeKind variant argument + Type(TypeKind), +} + +impl InputType { + /// Optionally unwraps as a PredicateForm. + pub fn predicate_form(&self) -> Option<&PredicateForm> { + match self { + InputType::PredicateForm(pf) => Some(pf), + _ => None, + } + } + + /// Optionally unwraps as a mutable PredicateForm + pub fn predicate_form_mut(&mut self) -> Option<&mut PredicateForm> { + match self { + InputType::PredicateForm(pf) => Some(pf), + _ => None, + } + } + + /// Optionally unwraps as a TypeKind. + pub fn typekind(&self) -> Option<&TypeKind> { + match self { + InputType::Type(ty) => Some(ty), + _ => None, + } + } + + /// Optionally unwraps as a NVariantOp + pub fn n_variant_op(&self) -> Option<&WildString> { + match self { + InputType::NVariantOp(Some(op)) => Some(op), + _ => None, + } + } +} + +impl PartialOrd for InputType { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for InputType { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + use std::cmp::Ordering::*; + + match (self, other) { + (InputType::PredicateForm(pf1), InputType::PredicateForm(pf2)) => pf1.cmp(pf2), + (InputType::Type(ty1), InputType::Type(ty2)) => ty1.cmp(ty2), + + (InputType::NVariantOp(None), InputType::NVariantOp(Some(..))) => Less, + (InputType::NVariantOp(Some(..)), InputType::NVariantOp(None)) => Greater, + (InputType::NVariantOp(_), InputType::NVariantOp(_)) => Equal, + + (InputType::Type(..), InputType::PredicateForm(..)) => Less, + (InputType::PredicateForm(..), InputType::Type(..)) => Greater, + + (InputType::Type(..), InputType::NVariantOp(..)) => Less, + (InputType::NVariantOp(..), InputType::Type(..)) => Greater, + + (InputType::PredicateForm(..), InputType::NVariantOp(..)) => Less, + (InputType::NVariantOp(..), InputType::PredicateForm(..)) => Greater, + } + } +} + +mod many_or_one { + use serde::{de::Deserializer, ser::Serializer, Deserialize, Serialize}; + + pub fn serialize(vec: &Vec, serializer: S) -> Result + where + T: Serialize, + S: Serializer, + { + if vec.len() == 1 { + vec.first().unwrap().serialize(serializer) + } else { + vec.serialize(serializer) + } + } + + pub fn deserialize<'de, T, D>(deserializer: D) -> Result, D::Error> + where + T: Deserialize<'de>, + D: Deserializer<'de>, + { + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(untagged)] + enum ManyOrOne { + Many(Vec), + One(T), + } + + match ManyOrOne::deserialize(deserializer)? { + ManyOrOne::Many(vec) => Ok(vec), + ManyOrOne::One(val) => Ok(vec![val]), + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct InputSet(#[serde(with = "many_or_one")] Vec); + +impl InputSet { + pub fn get(&self, idx: usize) -> Option<&InputType> { + self.0.get(idx) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator + '_ { + self.0.iter_mut() + } + + pub fn into_iter(self) -> impl Iterator + Clone { + self.0.into_iter() + } + + pub fn types_len(&self) -> usize { + self.iter().filter_map(|arg| arg.typekind()).count() + } + + pub fn typekind(&self, idx: Option) -> Option { + let types_len = self.types_len(); + self.get(idx.unwrap_or(0)).and_then(move |arg: &InputType| { + if (idx.is_none() && types_len != 1) || (idx.is_some() && types_len == 1) { + None + } else { + arg.typekind().cloned() + } + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InputSetEntry(#[serde(with = "many_or_one")] Vec); + +impl InputSetEntry { + pub fn new(input: Vec) -> Self { + Self(input) + } + + pub fn get(&self, idx: usize) -> Option<&InputSet> { + self.0.get(idx) + } +} + +fn validate_types<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let v: Vec = Vec::deserialize(deserializer)?; + + let mut it = v.iter(); + if let Some(first) = it.next() { + it.try_fold(first, |last, cur| { + if last.0.len() == cur.0.len() { + Ok(cur) + } else { + Err("the length of the InputSets and the product lists must match".to_string()) + } + }) + .map_err(de::Error::custom)?; + } + + Ok(v) +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct IntrinsicInput { + #[serde(default)] + #[serde(deserialize_with = "validate_types")] + pub types: Vec, + + #[serde(flatten)] + pub predication_methods: PredicationMethods, + + /// Generates a _n variant where the specified operand is a primitive type + /// that requires conversion to an SVE one. The `{_n}` wildcard is required + /// in the intrinsic's name, otherwise an error will be thrown. + #[serde(default)] + pub n_variant_op: WildString, +} + +impl IntrinsicInput { + /// Extracts all the possible variants as an iterator. + pub fn variants( + &self, + intrinsic: &Intrinsic, + ) -> context::Result + '_> { + let mut top_product = vec![]; + + if !self.types.is_empty() { + top_product.push( + self.types + .iter() + .flat_map(|ty_in| { + ty_in + .0 + .iter() + .map(|v| v.clone().into_iter()) + .multi_cartesian_product() + }) + .collect_vec(), + ) + } + + if let Ok(mask) = PredicationMask::try_from(&intrinsic.signature.name) { + top_product.push( + PredicateForm::compile_list(&mask, &self.predication_methods)? + .into_iter() + .map(|pf| vec![InputType::PredicateForm(pf)]) + .collect_vec(), + ) + } + + if !self.n_variant_op.is_empty() { + top_product.push(vec![ + vec![InputType::NVariantOp(None)], + vec![InputType::NVariantOp(Some(self.n_variant_op.to_owned()))], + ]) + } + + let it = top_product + .into_iter() + .map(|v| v.into_iter()) + .multi_cartesian_product() + .map(|set| InputSet(set.into_iter().flatten().collect_vec())); + Ok(it) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GeneratorInput { + #[serde(flatten)] + pub ctx: GlobalContext, + pub intrinsics: Vec, +} + +#[cfg(test)] +mod tests { + use crate::{ + input::*, + predicate_forms::{DontCareMethod, ZeroingMethod}, + }; + + #[test] + fn test_empty() { + let str = r#"types: []"#; + let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); + let mut variants = input.variants(&Intrinsic::default()).unwrap().into_iter(); + assert_eq!(variants.next(), None); + } + + #[test] + fn test_product() { + let str = r#"types: +- [f64, f32] +- [i64, [f64, f32]] +"#; + let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); + let mut intrinsic = Intrinsic::default(); + intrinsic.signature.name = "test_intrinsic{_mx}".parse().unwrap(); + let mut variants = input.variants(&intrinsic).unwrap().into_iter(); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("f64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::PredicateForm(PredicateForm::Merging), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("f64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::Type("f64".parse().unwrap()), + InputType::PredicateForm(PredicateForm::Merging), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::Type("f64".parse().unwrap()), + InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::PredicateForm(PredicateForm::Merging), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), + ])), + ); + assert_eq!(variants.next(), None); + } + + #[test] + fn test_n_variant() { + let str = r#"types: +- [f64, f32] +n_variant_op: op2 +"#; + let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); + let mut variants = input.variants(&Intrinsic::default()).unwrap().into_iter(); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("f64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::NVariantOp(None), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("f64".parse().unwrap()), + InputType::Type("f32".parse().unwrap()), + InputType::NVariantOp(Some("op2".parse().unwrap())), + ])) + ); + assert_eq!(variants.next(), None) + } + + #[test] + fn test_invalid_length() { + let str = r#"types: [i32, [[u64], [u32]]]"#; + serde_yaml::from_str::(str).expect_err("failure expected"); + } + + #[test] + fn test_invalid_predication() { + let str = "types: []"; + let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); + let mut intrinsic = Intrinsic::default(); + intrinsic.signature.name = "test_intrinsic{_mxz}".parse().unwrap(); + input + .variants(&intrinsic) + .map(|v| v.collect_vec()) + .expect_err("failure expected"); + } + + #[test] + fn test_invalid_predication_mask() { + "test_intrinsic{_mxy}" + .parse::() + .expect_err("failure expected"); + "test_intrinsic{_}" + .parse::() + .expect_err("failure expected"); + } + + #[test] + fn test_zeroing_predication() { + let str = r#"types: [i64] +zeroing_method: { drop: inactive }"#; + let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); + let mut intrinsic = Intrinsic::default(); + intrinsic.signature.name = "test_intrinsic{_mxz}".parse().unwrap(); + let mut variants = input.variants(&intrinsic).unwrap(); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::PredicateForm(PredicateForm::Merging), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsZeroing)), + ])) + ); + assert_eq!( + variants.next(), + Some(InputSet(vec![ + InputType::Type("i64".parse().unwrap()), + InputType::PredicateForm(PredicateForm::Zeroing(ZeroingMethod::Drop { + drop: "inactive".parse().unwrap() + })), + ])) + ); + assert_eq!(variants.next(), None) + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs b/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs new file mode 100644 index 00000000000..4a973691b17 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/intrinsic.rs @@ -0,0 +1,1624 @@ +use itertools::Itertools; +use proc_macro2::{Punct, Spacing, TokenStream}; +use quote::{format_ident, quote, ToTokens, TokenStreamExt}; +use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use std::collections::{HashMap, HashSet}; +use std::fmt::{self}; +use std::num::ParseIntError; +use std::ops::RangeInclusive; +use std::str::FromStr; + +use crate::assert_instr::InstructionAssertionsForBaseType; +use crate::context::{GlobalContext, GroupContext}; +use crate::input::{InputSet, InputSetEntry}; +use crate::predicate_forms::{DontCareMethod, PredicateForm, PredicationMask, ZeroingMethod}; +use crate::{ + assert_instr::InstructionAssertionMethod, + context::{self, ArchitectureSettings, Context, LocalContext, VariableType}, + expression::{Expression, FnCall, IdentifierType}, + fn_suffix::{type_to_size, SuffixKind}, + input::IntrinsicInput, + matching::{KindMatchable, SizeMatchable}, + typekinds::*, + wildcards::Wildcard, + wildstring::WildString, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum SubstitutionType { + MatchSize(SizeMatchable), + MatchKind(KindMatchable), +} + +impl SubstitutionType { + pub fn get(&mut self, ctx: &LocalContext) -> context::Result { + match self { + Self::MatchSize(smws) => { + smws.perform_match(ctx)?; + Ok(smws.as_ref().clone()) + } + Self::MatchKind(kmws) => { + kmws.perform_match(ctx)?; + Ok(kmws.as_ref().clone()) + } + } + } +} + +/// Mutability level +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum AccessLevel { + /// Immutable + R, + /// Mutable + RW, +} + +/// Function signature argument. +/// +/// Prepend the `mut` keyword for a mutable argument. Separate argument name +/// and type with a semicolon `:`. Usage examples: +/// - Mutable argument: `mut arg1: *u64` +/// - Immutable argument: `arg2: u32` +#[derive(Debug, Clone, SerializeDisplay, DeserializeFromStr)] +pub struct Argument { + /// Argument name + pub name: WildString, + /// Mutability level + pub rw: AccessLevel, + /// Argument type + pub kind: TypeKind, +} + +impl Argument { + pub fn populate_variables(&self, vars: &mut HashMap) { + vars.insert( + self.name.to_string(), + (self.kind.clone(), VariableType::Argument), + ); + } +} + +impl FromStr for Argument { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut it = s.splitn(2, ':').map(::trim); + if let Some(mut lhs) = it.next().map(|s| s.split_whitespace()) { + let lhs_len = lhs.clone().count(); + match (lhs_len, lhs.next(), it.next()) { + (2, Some("mut"), Some(kind)) => Ok(Argument { + name: lhs.next().unwrap().parse()?, + rw: AccessLevel::RW, + kind: kind.parse()?, + }), + (2, Some(ident), _) => Err(format!("invalid {ident:#?} keyword")), + (1, Some(name), Some(kind)) => Ok(Argument { + name: name.parse()?, + rw: AccessLevel::R, + kind: kind.parse()?, + }), + _ => Err(format!("invalid argument `{s}` provided")), + } + } else { + Err(format!("invalid argument `{s}` provided")) + } + } +} + +impl fmt::Display for Argument { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let AccessLevel::RW = &self.rw { + write!(f, "mut ")?; + } + + write!(f, "{}: {}", self.name, self.kind) + } +} + +impl ToTokens for Argument { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let AccessLevel::RW = &self.rw { + tokens.append(format_ident!("mut")) + } + + let (name, kind) = (format_ident!("{}", self.name.to_string()), &self.kind); + tokens.append_all(quote! { #name: #kind }) + } +} + +/// Static definition part of the signature. It may evaluate to a constant +/// expression with e.g. `const imm: u64`, or a generic `T: Into`. +#[derive(Debug, Clone, SerializeDisplay, DeserializeFromStr)] +pub enum StaticDefinition { + /// Constant expression + Constant(Argument), + /// Generic type + Generic(String), +} + +impl StaticDefinition { + pub fn as_variable(&self) -> Option<(String, (TypeKind, VariableType))> { + match self { + StaticDefinition::Constant(arg) => Some(( + arg.name.to_string(), + (arg.kind.clone(), VariableType::Argument), + )), + StaticDefinition::Generic(..) => None, + } + } +} + +impl FromStr for StaticDefinition { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim() { + s if s.starts_with("const ") => Ok(StaticDefinition::Constant(s[6..].trim().parse()?)), + s => Ok(StaticDefinition::Generic(s.to_string())), + } + } +} + +impl fmt::Display for StaticDefinition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + StaticDefinition::Constant(arg) => write!(f, "const {arg}"), + StaticDefinition::Generic(generic) => write!(f, "{generic}"), + } + } +} + +impl ToTokens for StaticDefinition { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(match self { + StaticDefinition::Constant(arg) => quote! { const #arg }, + StaticDefinition::Generic(generic) => { + let generic: TokenStream = generic.parse().expect("invalid Rust code"); + quote! { #generic } + } + }) + } +} + +/// Function constraints +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Constraint { + /// Asserts that the given variable equals to any of the given integer values + AnyI32 { + variable: String, + any_values: Vec, + }, + /// WildString version of RangeI32. If the string values given for the range + /// are valid, this gets built into a RangeI32. + RangeWildstring { + variable: String, + range: (WildString, WildString), + }, + /// Asserts that the given variable's value falls in the specified range + RangeI32 { + variable: String, + range: SizeMatchable>, + }, + /// Asserts that the number of elements/lanes does not exceed the 2048-bit SVE constraint + SVEMaxElems { + variable: String, + sve_max_elems_type: TypeKind, + }, + /// Asserts that the number of elements/lanes does not exceed the 128-bit register constraint + VecMaxElems { + variable: String, + vec_max_elems_type: TypeKind, + }, +} + +impl Constraint { + fn variable(&self) -> &str { + match self { + Constraint::AnyI32 { variable, .. } + | Constraint::RangeWildstring { variable, .. } + | Constraint::RangeI32 { variable, .. } + | Constraint::SVEMaxElems { variable, .. } + | Constraint::VecMaxElems { variable, .. } => variable, + } + } + pub fn build(&mut self, ctx: &Context) -> context::Result { + if let Self::RangeWildstring { + variable, + range: (min, max), + } = self + { + min.build_acle(ctx.local)?; + max.build_acle(ctx.local)?; + let min = min.to_string(); + let max = max.to_string(); + let min: i32 = min + .parse() + .map_err(|_| format!("the minimum value `{min}` is not a valid number"))?; + let max: i32 = max + .parse() + .or_else(|_| Ok(type_to_size(max.as_str()))) + .map_err(|_: ParseIntError| { + format!("the maximum value `{max}` is not a valid number") + })?; + *self = Self::RangeI32 { + variable: variable.to_owned(), + range: SizeMatchable::Matched(RangeInclusive::new(min, max)), + } + } + + if let Self::SVEMaxElems { + sve_max_elems_type: ty, + .. + } + | Self::VecMaxElems { + vec_max_elems_type: ty, + .. + } = self + { + if let Some(w) = ty.wildcard() { + ty.populate_wildcard(ctx.local.provide_type_wildcard(w)?)?; + } + } + + if let Self::RangeI32 { range, .. } = self { + range.perform_match(ctx.local)?; + } + + let variable = self.variable(); + ctx.local + .variables + .contains_key(variable) + .then_some(()) + .ok_or_else(|| format!("cannot build constraint, could not find variable {variable}")) + } +} + +/// Function signature +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Signature { + /// Function name + pub name: WildString, + /// List of function arguments, leave unset or empty for no arguments + pub arguments: Vec, + /// Function return type, leave unset for void + pub return_type: Option, + + /// For some neon intrinsics we want to modify the suffix of the function name + pub suffix_type: Option, + + /// List of static definitions, leave unset of empty if not required + #[serde(default)] + pub static_defs: Vec, + + /// **Internal use only.** + /// Condition for which the ultimate function is specific to predicates. + #[serde(skip)] + pub is_predicate_specific: bool, + + /// **Internal use only.** + /// Setting this property will trigger the signature builder to convert any `svbool*_t` to `svbool_t` in the input and output. + #[serde(skip)] + pub predicate_needs_conversion: bool, +} + +impl Signature { + pub fn drop_argument(&mut self, arg_name: &WildString) -> Result<(), String> { + if let Some(idx) = self + .arguments + .iter() + .position(|arg| arg.name.to_string() == arg_name.to_string()) + { + self.arguments.remove(idx); + Ok(()) + } else { + Err(format!("no argument {arg_name} found to drop")) + } + } + + pub fn build(&mut self, ctx: &LocalContext) -> context::Result { + if self.name_has_neon_suffix() { + self.name.build_neon_intrinsic_signature(ctx)?; + } else { + self.name.build_acle(ctx)?; + } + + if let Some(ref mut return_type) = self.return_type { + if let Some(w) = return_type.clone().wildcard() { + return_type.populate_wildcard(ctx.provide_type_wildcard(w)?)?; + } + } + + self.arguments + .iter_mut() + .try_for_each(|arg| arg.name.build_acle(ctx))?; + + self.arguments + .iter_mut() + .filter_map(|arg| { + arg.kind + .clone() + .wildcard() + .map(|w| (&mut arg.kind, w.clone())) + }) + .try_for_each(|(ty, w)| ty.populate_wildcard(ctx.provide_type_wildcard(&w)?)) + } + + pub fn fn_name(&self) -> WildString { + self.name.replace(['[', ']'], "") + } + + pub fn doc_name(&self) -> String { + self.name.to_string() + } + + fn name_has_neon_suffix(&self) -> bool { + for part in self.name.wildcards() { + let has_suffix = match part { + Wildcard::NEONType(_, _, suffix_type) => suffix_type.is_some(), + _ => false, + }; + + if has_suffix { + return true; + } + } + false + } +} + +impl ToTokens for Signature { + fn to_tokens(&self, tokens: &mut TokenStream) { + let name_ident = format_ident!("{}", self.fn_name().to_string()); + let arguments = self + .arguments + .clone() + .into_iter() + .map(|mut arg| { + if arg + .kind + .vector() + .map_or(false, |ty| ty.base_type().is_bool()) + && self.predicate_needs_conversion + { + arg.kind = TypeKind::Vector(VectorType::make_predicate_from_bitsize(8)) + } + arg + }) + .collect_vec(); + let static_defs = &self.static_defs; + tokens.append_all(quote! { fn #name_ident<#(#static_defs),*>(#(#arguments),*) }); + + if let Some(ref return_type) = self.return_type { + if return_type + .vector() + .map_or(false, |ty| ty.base_type().is_bool()) + && self.predicate_needs_conversion + { + tokens.append_all(quote! { -> svbool_t }) + } else { + tokens.append_all(quote! { -> #return_type }) + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LLVMLinkAttribute { + /// Either one architecture or a comma separated list of architectures with NO spaces + pub arch: String, + pub link: WildString, +} + +impl ToTokens for LLVMLinkAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + let LLVMLinkAttribute { arch, link } = self; + let link = link.to_string(); + let archs: Vec<&str> = arch.split(',').collect(); + let arch_len = archs.len(); + + if arch_len == 1 { + tokens.append_all(quote! { + #[cfg_attr(target_arch = #arch, link_name = #link)] + }) + } else { + tokens.append(Punct::new('#', Spacing::Alone)); + tokens.append(Punct::new('[', Spacing::Alone)); + tokens.append_all(quote! { cfg_attr }); + tokens.append(Punct::new('(', Spacing::Alone)); + tokens.append_all(quote! { any }); + tokens.append(Punct::new('(', Spacing::Alone)); + let mut i = 0; + while i < arch_len { + let arch = archs[i].to_string(); + tokens.append_all(quote! { target_arch = #arch }); + if i + 1 != arch_len { + tokens.append(Punct::new(',', Spacing::Alone)); + } + i += 1; + } + tokens.append(Punct::new(')', Spacing::Alone)); + tokens.append(Punct::new(',', Spacing::Alone)); + tokens.append_all(quote! { link_name = #link }); + tokens.append(Punct::new(')', Spacing::Alone)); + tokens.append(Punct::new(']', Spacing::Alone)); + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LLVMLink { + /// LLVM link function name without namespace and types, + /// e.g. `st1` in `llvm.aarch64.sve.st1.nxv4i32` + pub name: WildString, + + /// LLVM link signature arguments, leave unset if it inherits from intrinsic's signature + pub arguments: Option>, + /// LLVM link signature return type, leave unset if it inherits from intrinsic's signature + pub return_type: Option, + + /// **This will be set automatically if not set** + /// Attribute LLVM links for the function. First element is the architecture it targets, + /// second element is the LLVM link itself. + pub links: Option>, + + /// **Internal use only. Do not set.** + /// Generated signature from these `arguments` and/or `return_type` if set, and the intrinsic's signature. + #[serde(skip)] + pub signature: Option>, +} + +impl LLVMLink { + pub fn resolve(&self, cfg: &ArchitectureSettings) -> String { + self.name + .starts_with("llvm") + .then(|| self.name.to_string()) + .unwrap_or_else(|| format!("{}.{}", cfg.llvm_link_prefix, self.name)) + } + + pub fn build_and_save(&mut self, ctx: &mut Context) -> context::Result { + self.build(ctx)?; + + // Save LLVM link to the group context + ctx.global.arch_cfgs.iter().for_each(|cfg| { + ctx.group + .links + .insert(self.resolve(cfg), ctx.local.input.clone()); + }); + + Ok(()) + } + + pub fn build(&mut self, ctx: &mut Context) -> context::Result { + let mut sig_name = ctx.local.signature.name.clone(); + sig_name.prepend_str("_"); + + let mut sig = Signature { + name: sig_name, + arguments: self + .arguments + .clone() + .unwrap_or_else(|| ctx.local.signature.arguments.clone()), + return_type: self + .return_type + .clone() + .or_else(|| ctx.local.signature.return_type.clone()), + suffix_type: None, + static_defs: vec![], + is_predicate_specific: ctx.local.signature.is_predicate_specific, + predicate_needs_conversion: false, + }; + + sig.build(ctx.local)?; + self.name.build(ctx.local, TypeRepr::LLVMMachine)?; + + // Add link function name to context + ctx.local + .substitutions + .insert(Wildcard::LLVMLink, sig.fn_name().to_string()); + + self.signature = Some(Box::new(sig)); + + if let Some(ref mut links) = self.links { + links.iter_mut().for_each(|ele| { + ele.link + .build(&ctx.local, TypeRepr::LLVMMachine) + .expect("Failed to transform to LLVMMachine representation"); + }); + } else { + self.links = Some( + ctx.global + .arch_cfgs + .iter() + .map(|cfg| LLVMLinkAttribute { + arch: cfg.arch_name.to_owned(), + link: self.resolve(cfg).into(), + }) + .collect_vec(), + ); + } + + Ok(()) + } + + /// Alters all the unsigned types from the signature, as unsupported by LLVM. + pub fn sanitise_uints(&mut self) { + let transform = |tk: &mut TypeKind| { + if let Some(BaseType::Sized(BaseTypeKind::UInt, size)) = tk.base_type() { + *tk.base_type_mut().unwrap() = BaseType::Sized(BaseTypeKind::Int, *size) + } + }; + + if let Some(sig) = self.signature.as_mut() { + for arg in sig.arguments.iter_mut() { + transform(&mut arg.kind); + } + + sig.return_type.as_mut().map(transform); + } + } + + /// Make a function call to the LLVM link + pub fn make_fn_call(&self, intrinsic_sig: &Signature) -> context::Result { + let link_sig = self.signature.as_ref().ok_or_else(|| { + "cannot derive the LLVM link call, as it does not hold a valid function signature" + .to_string() + })?; + + if intrinsic_sig.arguments.len() != link_sig.arguments.len() { + return Err( + "cannot derive the LLVM link call, the number of arguments does not match" + .to_string(), + ); + } + + let call_args = intrinsic_sig + .arguments + .iter() + .zip(link_sig.arguments.iter()) + .map(|(intrinsic_arg, link_arg)| { + // Could also add a type check... + if intrinsic_arg.name == link_arg.name { + Ok(Expression::Identifier( + intrinsic_arg.name.to_owned(), + IdentifierType::Variable, + )) + } else { + Err("cannot derive the LLVM link call, the arguments do not match".to_string()) + } + }) + .try_collect()?; + + Ok(FnCall::new_expression(link_sig.fn_name().into(), call_args)) + } + + /// Given a FnCall, apply all the predicate and unsigned conversions as required. + pub fn apply_conversions_to_call( + &self, + mut fn_call: FnCall, + ctx: &LocalContext, + ) -> context::Result { + use BaseType::{Sized, Unsized}; + use BaseTypeKind::{Bool, UInt}; + use VariableType::Argument; + + let convert = + |method: &str, ex| Expression::MethodCall(Box::new(ex), method.to_string(), vec![]); + + fn_call.1 = fn_call + .1 + .into_iter() + .map(|arg| -> context::Result { + if let Expression::Identifier(ref var_name, IdentifierType::Variable) = arg { + let (kind, scope) = ctx + .variables + .get(&var_name.to_string()) + .ok_or_else(|| format!("invalid variable {var_name:?} being referenced"))?; + + match (scope, kind.base_type()) { + (Argument, Some(Sized(Bool, bitsize))) if *bitsize != 8 => { + Ok(convert("into", arg)) + } + (Argument, Some(Sized(UInt, _) | Unsized(UInt))) => { + Ok(convert("as_signed", arg)) + } + _ => Ok(arg), + } + } else { + Ok(arg) + } + }) + .try_collect()?; + + let return_type_requires_conversion = self + .signature + .as_ref() + .and_then(|sig| sig.return_type.as_ref()) + .and_then(|ty| { + if let Some(Sized(Bool, bitsize)) = ty.base_type() { + (*bitsize != 8).then_some(Bool) + } else if let Some(Sized(UInt, _) | Unsized(UInt)) = ty.base_type() { + Some(UInt) + } else { + None + } + }); + + let fn_call = Expression::FnCall(fn_call); + match return_type_requires_conversion { + Some(Bool) => Ok(convert("into", fn_call)), + Some(UInt) => Ok(convert("as_unsigned", fn_call)), + _ => Ok(fn_call), + } + } +} + +impl ToTokens for LLVMLink { + fn to_tokens(&self, tokens: &mut TokenStream) { + assert!( + self.signature.is_some() && self.links.is_some(), + "expression {self:#?} was not built before calling to_tokens" + ); + + let signature = self.signature.as_ref().unwrap(); + let links = self.links.as_ref().unwrap(); + tokens.append_all(quote! { + extern "unadjusted" { + #(#links)* + #signature; + } + }) + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FunctionVisibility { + #[default] + Public, + Private, +} + +/// Whether to generate a load/store test, and which typeset index +/// represents the data type of the load/store target address +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Test { + #[default] + #[serde(skip)] + None, // Covered by `intrinsic-test` + Load(usize), + Store(usize), +} + +impl Test { + pub fn get_typeset_index(&self) -> Option { + match *self { + Test::Load(n) => Some(n), + Test::Store(n) => Some(n), + _ => None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Safety { + Safe, + Unsafe(Vec), +} + +impl Safety { + /// Return `Ok(Safety::Safe)` if safety appears reasonable for the given `intrinsic`'s name and + /// prototype. Otherwise, return `Err()` with a suitable diagnostic. + fn safe_checked(intrinsic: &Intrinsic) -> Result { + let name = intrinsic.signature.doc_name(); + if name.starts_with("sv") { + let handles_pointers = intrinsic + .signature + .arguments + .iter() + .any(|arg| matches!(arg.kind, TypeKind::Pointer(..))); + if name.starts_with("svld") + || name.starts_with("svst") + || name.starts_with("svprf") + || name.starts_with("svundef") + || handles_pointers + { + let doc = intrinsic.doc.as_ref().map(|s| s.to_string()); + let doc = doc.as_deref().unwrap_or("..."); + Err(format!( + "`{name}` has no safety specification, but it looks like it should be unsafe. \ + Consider specifying (un)safety explicitly: + + - name: {name} + doc: {doc} + safety: + unsafe: + - ... + ... +" + )) + } else { + Ok(Self::Safe) + } + } else { + Err(format!( + "Safety::safe_checked() for non-SVE intrinsic: {name}" + )) + } + } + + fn is_safe(&self) -> bool { + match self { + Self::Safe => true, + Self::Unsafe(..) => false, + } + } + + fn is_unsafe(&self) -> bool { + !self.is_safe() + } + + fn has_doc_comments(&self) -> bool { + match self { + Self::Safe => false, + Self::Unsafe(v) => !v.is_empty(), + } + } + + fn doc_comments(&self) -> &[UnsafetyComment] { + match self { + Self::Safe => &[], + Self::Unsafe(v) => v.as_slice(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UnsafetyComment { + Custom(String), + Uninitialized, + PointerOffset(GovernedBy), + PointerOffsetVnum(GovernedBy), + Dereference(GovernedBy), + UnpredictableOnFault, + NonTemporal, + Neon, + NoProvenance(String), +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GovernedBy { + #[default] + Predicated, + PredicatedNonFaulting, + PredicatedFirstFaulting, +} + +impl fmt::Display for GovernedBy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Predicated => write!(f, " (governed by `pg`)"), + Self::PredicatedNonFaulting => write!( + f, + " (governed by `pg`, the first-fault register (`FFR`) \ + and non-faulting behaviour)" + ), + Self::PredicatedFirstFaulting => write!( + f, + " (governed by `pg`, the first-fault register (`FFR`) \ + and first-faulting behaviour)" + ), + } + } +} + +impl fmt::Display for UnsafetyComment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Custom(s) => s.fmt(f), + Self::Neon => write!(f, "Neon instrinsic unsafe"), + Self::Uninitialized => write!( + f, + "This creates an uninitialized value, and may be unsound (like \ + [`core::mem::uninitialized`])." + ), + Self::PointerOffset(gov) => write!( + f, + "[`pointer::offset`](pointer#method.offset) safety constraints must \ + be met for the address calculation for each active element{gov}." + ), + Self::PointerOffsetVnum(gov) => write!( + f, + "[`pointer::offset`](pointer#method.offset) safety constraints must \ + be met for the address calculation for each active element{gov}. \ + In particular, note that `vnum` is scaled by the vector \ + length, `VL`, which is not known at compile time." + ), + Self::Dereference(gov) => write!( + f, + "This dereferences and accesses the calculated address for each \ + active element{gov}." + ), + Self::NonTemporal => write!( + f, + "Non-temporal accesses have special memory ordering rules, and \ + [explicit barriers may be required for some applications]\ + (https://developer.arm.com/documentation/den0024/a/Memory-Ordering/Barriers/Non-temporal-load-and-store-pair?lang=en)." + ), + Self::NoProvenance(arg) => write!( + f, + "Addresses passed in `{arg}` lack provenance, so this is similar to using a \ + `usize as ptr` cast (or [`core::ptr::from_exposed_addr`]) on each lane before \ + using it." + ), + Self::UnpredictableOnFault => write!( + f, + "Result lanes corresponding to inactive FFR lanes (either before or as a result \ + of this intrinsic) have \"CONSTRAINED UNPREDICTABLE\" values, irrespective of \ + predication. Refer to architectural documentation for details." + ), + } + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Intrinsic { + #[serde(default)] + pub visibility: FunctionVisibility, + #[serde(default)] + pub doc: Option, + #[serde(flatten)] + pub signature: Signature, + /// Function sequential composition + pub compose: Vec, + /// Input to generate the intrinsic against. Leave empty if the intrinsic + /// does not have any variants. + /// Specific variants contain one InputSet + #[serde(flatten, default)] + pub input: IntrinsicInput, + #[serde(default)] + pub constraints: Vec, + /// Additional target features to add to the global settings + #[serde(default)] + pub target_features: Vec, + /// Should the intrinsic be `unsafe`? By default, the generator will try to guess from the + /// prototype, but it errs on the side of `unsafe`, and prints a warning in that case. + #[serde(default)] + pub safety: Option, + #[serde(default)] + pub substitutions: HashMap, + /// List of the only indices in a typeset that require conversion to signed + /// when deferring unsigned intrinsics to signed. (optional, default + /// behaviour is all unsigned types are converted to signed) + #[serde(default)] + pub defer_to_signed_only_indices: HashSet, + pub assert_instr: Option>, + /// Whether we should generate a test for this intrinsic + #[serde(default)] + pub test: Test, + /// Primary base type, used for instruction assertion. + #[serde(skip)] + pub base_type: Option, + /// Attributes for the function + pub attr: Option>, +} + +impl Intrinsic { + pub fn llvm_link(&self) -> Option<&LLVMLink> { + self.compose.iter().find_map(|ex| { + if let Expression::LLVMLink(llvm_link) = ex { + Some(llvm_link) + } else { + None + } + }) + } + + pub fn llvm_link_mut(&mut self) -> Option<&mut LLVMLink> { + self.compose.iter_mut().find_map(|ex| { + if let Expression::LLVMLink(llvm_link) = ex { + Some(llvm_link) + } else { + None + } + }) + } + + pub fn generate_variants(&self, global_ctx: &GlobalContext) -> context::Result> { + let wrap_err = |err| format!("{}: {err}", self.signature.name); + + let mut group_ctx = GroupContext::default(); + self.input + .variants(self) + .map_err(wrap_err)? + .map(|input| { + self.generate_variant(input.clone(), &mut group_ctx, global_ctx) + .map_err(wrap_err) + .map(|variant| (variant, input)) + }) + .collect::>>() + .and_then(|mut variants| { + variants.sort_by_cached_key(|(_, input)| input.to_owned()); + + if variants.is_empty() { + let standalone_variant = self + .generate_variant(InputSet::default(), &mut group_ctx, global_ctx) + .map_err(wrap_err)?; + + Ok(vec![standalone_variant]) + } else { + Ok(variants + .into_iter() + .map(|(variant, _)| variant) + .collect_vec()) + } + }) + } + + pub fn generate_variant( + &self, + input: InputSet, + group_ctx: &mut GroupContext, + global_ctx: &GlobalContext, + ) -> context::Result { + let mut variant = self.clone(); + + variant.input.types = vec![InputSetEntry::new(vec![input.clone()])]; + + let mut local_ctx = LocalContext::new(input, self); + let mut ctx = Context { + local: &mut local_ctx, + group: group_ctx, + global: global_ctx, + }; + + variant.pre_build(&mut ctx)?; + + match ctx.local.predicate_form().cloned() { + Some(PredicateForm::DontCare(method)) => { + variant.compose = variant.generate_dont_care_pass_through(&mut ctx, method)? + } + Some(PredicateForm::Zeroing(method)) => { + variant.compose = variant.generate_zeroing_pass_through(&mut ctx, method)? + } + _ => { + for idx in 0..variant.compose.len() { + let mut ex = variant.compose[idx].clone(); + ex.build(&variant, &mut ctx)?; + variant.compose[idx] = ex; + } + } + }; + + if variant.attr.is_none() && variant.assert_instr.is_none() { + panic!("Error: {} is missing both 'attr' and 'assert_instr' fields. You must either manually declare the attributes using the 'attr' field or use 'assert_instr'!", variant.signature.name.to_string()); + } + + if variant.attr.is_some() { + let attr: &Vec = &variant.attr.clone().unwrap(); + let mut expanded_attr: Vec = Vec::new(); + for idx in 0..attr.len() { + let mut ex = attr[idx].clone(); + ex.build(&variant, &mut ctx)?; + expanded_attr.push(ex); + } + variant.attr = Some(expanded_attr); + } + + variant.post_build(&mut ctx)?; + + if let Some(n_variant_op) = ctx.local.n_variant_op().cloned() { + variant.generate_n_variant(n_variant_op, &mut ctx) + } else { + Ok(variant) + } + } + + /// Implement a "zeroing" (_z) method by calling an existing "merging" (_m) method, as required. + fn generate_zeroing_pass_through( + &mut self, + ctx: &mut Context, + method: ZeroingMethod, + ) -> context::Result> { + PredicationMask::try_from(&ctx.local.signature.name) + .ok() + .filter(|mask| mask.has_merging()) + .ok_or_else(|| format!("cannot generate zeroing passthrough for {}, no merging predicate form is specified", self.signature.name))?; + + // Determine the function to pass through to. + let mut target_ctx = ctx.local.clone(); + // Change target function predicate form to merging + *target_ctx.input.iter_mut() + .find_map(|arg| arg.predicate_form_mut()) + .expect("failed to generate zeroing pass through, could not find predicate form in the InputSet") = PredicateForm::Merging; + + let mut sig = target_ctx.signature.clone(); + sig.build(&target_ctx)?; + + let args_as_expressions = |arg: &Argument| -> context::Result { + let arg_name = arg.name.to_string(); + match &method { + ZeroingMethod::Drop { drop } if arg_name == drop.to_string() => { + Ok(PredicateForm::make_zeroinitializer(&arg.kind)) + } + ZeroingMethod::Select { select } if arg_name == select.to_string() => { + let pg = sig + .arguments + .iter() + .find_map(|arg| match arg.kind.vector() { + Some(ty) if ty.base_type().is_bool() => Some(arg.name.clone()), + _ => None, + }) + .ok_or_else(|| { + format!("cannot generate zeroing passthrough for {}, no predicate found in the signature for zero selection", self.signature.name) + })?; + Ok(PredicateForm::make_zeroselector( + pg, + select.clone(), + &arg.kind, + )) + } + _ => Ok(arg.into()), + } + }; + + let name: Expression = sig.fn_name().into(); + let args: Vec = sig + .arguments + .iter() + .map(args_as_expressions) + .try_collect()?; + let statics: Vec = sig + .static_defs + .iter() + .map(|sd| sd.try_into()) + .try_collect()?; + let mut call: Expression = FnCall(Box::new(name), args, statics).into(); + call.build(self, ctx)?; + Ok(vec![call]) + } + + /// Implement a "don't care" (_x) method by calling an existing "merging" (_m). + fn generate_dont_care_pass_through( + &mut self, + ctx: &mut Context, + method: DontCareMethod, + ) -> context::Result> { + PredicationMask::try_from(&ctx.local.signature.name).and_then(|mask| match method { + DontCareMethod::AsMerging if mask.has_merging() => Ok(()), + DontCareMethod::AsZeroing if mask.has_zeroing() => Ok(()), + _ => Err(format!( + "cannot generate don't care passthrough for {}, no {} predicate form is specified", + self.signature.name, + match method { + DontCareMethod::AsMerging => "merging", + DontCareMethod::AsZeroing => "zeroing", + _ => unreachable!(), + } + )), + })?; + + // Determine the function to pass through to. + let mut target_ctx = ctx.local.clone(); + // Change target function predicate form to merging + *target_ctx.input.iter_mut() + .find_map(|arg| arg.predicate_form_mut()) + .expect("failed to generate don't care passthrough, could not find predicate form in the InputSet") = PredicateForm::Merging; + + let mut sig = target_ctx.signature.clone(); + sig.build(&target_ctx)?; + + // We might need to drop an argument for a zeroing pass-through. + let drop = match (method, &self.input.predication_methods.zeroing_method) { + (DontCareMethod::AsZeroing, Some(ZeroingMethod::Drop { drop })) => Some(drop), + _ => None, + }; + + let name: Expression = sig.fn_name().into(); + let args: Vec = sig + .arguments + .iter() + .map(|arg| { + if Some(arg.name.to_string()) == drop.as_ref().map(|v| v.to_string()) { + // This argument is present in the _m form, but missing from the _x form. Clang + // typically replaces these with an uninitialised vector, but to avoid + // materialising uninitialised values in Rust, we instead merge with a known + // vector. This usually results in the same code generation. + // TODO: In many cases, it'll be better to use an unpredicated (or zeroing) form. + sig.arguments + .iter() + .filter(|&other| arg.name.to_string() != other.name.to_string()) + .find_map(|other| { + arg.kind.express_reinterpretation_from(&other.kind, other) + }) + .unwrap_or_else(|| PredicateForm::make_zeroinitializer(&arg.kind)) + } else { + arg.into() + } + }) + .collect(); + let statics: Vec = sig + .static_defs + .iter() + .map(|sd| sd.try_into()) + .try_collect()?; + let mut call: Expression = FnCall(Box::new(name), args, statics).into(); + call.build(self, ctx)?; + Ok(vec![call]) + } + + /// Implement a "_n" variant based on the given operand + fn generate_n_variant( + &self, + mut n_variant_op: WildString, + ctx: &mut Context, + ) -> context::Result { + let mut variant = self.clone(); + + n_variant_op.build_acle(ctx.local)?; + + let n_op_arg_idx = variant + .signature + .arguments + .iter_mut() + .position(|arg| arg.name.to_string() == n_variant_op.to_string()) + .ok_or_else(|| { + format!( + "cannot generate `_n` variant for {}, operand `{n_variant_op}` not found", + variant.signature.name + ) + })?; + + let has_n_wildcard = ctx + .local + .signature + .name + .wildcards() + .any(|w| matches!(w, Wildcard::NVariant)); + + if !has_n_wildcard { + return Err(format!("cannot generate `_n` variant for {}, no wildcard {{_n}} was specified in the intrinsic's name", variant.signature.name)); + } + + // Build signature + variant.signature = ctx.local.signature.clone(); + if let Some(pf) = ctx.local.predicate_form() { + // WARN: this may break in the future according to the underlying implementation + // Drops unwanted arguments if needed (required for the collection of arguments to pass to the function) + pf.post_build(&mut variant)?; + } + + let sig = &mut variant.signature; + + ctx.local + .substitutions + .insert(Wildcard::NVariant, "_n".to_owned()); + + let arg_kind = &mut sig.arguments.get_mut(n_op_arg_idx).unwrap().kind; + *arg_kind = match arg_kind { + TypeKind::Wildcard(Wildcard::SVEType(idx, None)) => { + TypeKind::Wildcard(Wildcard::Type(*idx)) + } + _ => { + return Err(format!( + "cannot generate `_n` variant for {}, the given operand is not a valid SVE type", + variant.signature.name + )) + } + }; + + sig.build(ctx.local)?; + + // Build compose + let name: Expression = self.signature.fn_name().into(); + let args: Vec = sig + .arguments + .iter() + .enumerate() + .map(|(idx, arg)| { + let ty = arg.kind.acle_notation_repr(); + if idx == n_op_arg_idx { + FnCall::new_expression( + WildString::from(format!("svdup_n_{ty}")).into(), + vec![arg.into()], + ) + } else { + arg.into() + } + }) + .collect(); + let statics: Vec = sig + .static_defs + .iter() + .map(|sd| sd.try_into()) + .try_collect()?; + let mut call: Expression = FnCall(Box::new(name), args, statics).into(); + call.build(self, ctx)?; + + variant.compose = vec![call]; + variant.signature.predicate_needs_conversion = true; + + Ok(variant) + } + + fn pre_build(&mut self, ctx: &mut Context) -> context::Result { + self.substitutions + .iter_mut() + .try_for_each(|(k, v)| -> context::Result { + let mut ws = v.get(ctx.local)?; + ws.build_acle(ctx.local)?; + ctx.local + .substitutions + .insert(Wildcard::Custom(k.to_owned()), ws.to_string()); + Ok(()) + })?; + + self.signature.build(ctx.local)?; + + if self.safety.is_none() { + self.safety = match Safety::safe_checked(self) { + Ok(safe) => Some(safe), + Err(err) => { + eprintln!("{err}"); + return Err(format!( + "Refusing to infer unsafety for {name}", + name = self.signature.doc_name() + )); + } + } + } + + if let Some(doc) = &mut self.doc { + doc.build_acle(ctx.local)? + } + + // Add arguments to variable tracking + self.signature + .arguments + .iter() + .for_each(|arg| arg.populate_variables(&mut ctx.local.variables)); + + // Add constant expressions to variable tracking + self.signature + .static_defs + .iter() + .filter_map(StaticDefinition::as_variable) + .for_each(|(var_name, var_properties)| { + ctx.local.variables.insert(var_name, var_properties); + }); + + // Pre-build compose expressions + for idx in 0..self.compose.len() { + let mut ex = self.compose[idx].clone(); + ex.pre_build(ctx)?; + self.compose[idx] = ex; + } + + if !ctx.local.input.is_empty() { + // We simplify the LLVM link transmute logic by deferring to a variant employing the same LLVM link where possible + if let Some(link) = self.compose.iter().find_map(|ex| match ex { + Expression::LLVMLink(link) => Some(link), + _ => None, + }) { + let mut link = link.clone(); + link.build(ctx)?; + + for cfg in ctx.global.arch_cfgs.iter() { + let expected_link = link.resolve(cfg); + if let Some(target_inputset) = ctx.group.links.get(&expected_link) { + self.defer_to_existing_llvm_link(ctx.local, target_inputset)?; + break; + } + } + } + } + + if let Some(ref mut assert_instr) = self.assert_instr { + assert_instr.iter_mut().try_for_each(|ai| ai.build(ctx))?; + } + + // Prepend constraint assertions + self.constraints.iter_mut().try_for_each(|c| c.build(ctx))?; + let assertions: Vec<_> = self + .constraints + .iter() + .map(|c| ctx.local.make_assertion_from_constraint(c)) + .try_collect()?; + self.compose.splice(0..0, assertions); + + Ok(()) + } + + fn post_build(&mut self, ctx: &mut Context) -> context::Result { + if let Some(Expression::LLVMLink(link)) = self.compose.last() { + let mut fn_call = link.make_fn_call(&self.signature)?; + // Required to inject conversions + fn_call.build(self, ctx)?; + self.compose.push(fn_call) + } + + if let Some(llvm_link) = self.llvm_link_mut() { + // Turn all Rust unsigned types into signed + llvm_link.sanitise_uints(); + } + + if let Some(predicate_form) = ctx.local.predicate_form() { + predicate_form.post_build(self)? + } + + // Set for ToTokens to display a generic svbool_t + self.signature.predicate_needs_conversion = true; + + // Set base type kind for instruction assertion + self.base_type = ctx + .local + .input + .get(0) + .and_then(|arg| arg.typekind()) + .and_then(|ty| ty.base_type()) + .map(BaseType::clone); + + // Add global target features + self.target_features = ctx + .global + .arch_cfgs + .iter() + .flat_map(|cfg| cfg.target_feature.clone()) + .chain(self.target_features.clone()) + .collect_vec(); + + Ok(()) + } + + fn defer_to_existing_llvm_link( + &mut self, + ctx: &LocalContext, + target_inputset: &InputSet, + ) -> context::Result { + let mut target_ctx = ctx.clone(); + target_ctx.input = target_inputset.clone(); + + let mut target_signature = target_ctx.signature.clone(); + target_signature.build(&target_ctx)?; + + let drop_var = if let Some(pred) = ctx.predicate_form().cloned() { + match pred { + PredicateForm::Zeroing(ZeroingMethod::Drop { drop }) => Some(drop), + PredicateForm::DontCare(DontCareMethod::AsZeroing) => { + if let Some(ZeroingMethod::Drop { drop }) = + self.input.predication_methods.zeroing_method.to_owned() + { + Some(drop) + } else { + None + } + } + _ => None, + } + } else { + None + }; + + let call_method = + |ex, method: &str| Expression::MethodCall(Box::new(ex), method.to_string(), vec![]); + let as_unsigned = |ex| call_method(ex, "as_unsigned"); + let as_signed = |ex| call_method(ex, "as_signed"); + let convert_if_required = |w: Option<&Wildcard>, from: &InputSet, to: &InputSet, ex| { + if let Some(w) = w { + if let Some(dest_idx) = w.get_typeset_index() { + let from_type = from.get(dest_idx); + let to_type = to.get(dest_idx); + + if from_type != to_type { + let from_base_type = from_type + .and_then(|in_arg| in_arg.typekind()) + .and_then(|ty| ty.base_type()) + .map(|bt| bt.kind()); + let to_base_type = to_type + .and_then(|in_arg| in_arg.typekind()) + .and_then(|ty| ty.base_type()) + .map(|bt| bt.kind()); + + match (from_base_type, to_base_type) { + // Use AsSigned for uint -> int + (Some(BaseTypeKind::UInt), Some(BaseTypeKind::Int)) => as_signed(ex), + (Some(BaseTypeKind::Int), Some(BaseTypeKind::Int)) => ex, + // Use AsUnsigned for int -> uint + (Some(BaseTypeKind::Int), Some(BaseTypeKind::UInt)) => as_unsigned(ex), + (Some(BaseTypeKind::Float), Some(BaseTypeKind::Float)) => ex, + (Some(BaseTypeKind::UInt), Some(BaseTypeKind::UInt)) => ex, + (Some(BaseTypeKind::Poly), Some(BaseTypeKind::Poly)) => ex, + + (None, None) => ex, + _ => unreachable!("unsupported conversion case from {from_base_type:?} to {to_base_type:?} hit"), + } + } else { + ex + } + } else { + ex + } + } else { + ex + } + }; + + let args = ctx + .signature + .arguments + .iter() + .filter_map(|arg| { + let var = Expression::Identifier(arg.name.to_owned(), IdentifierType::Variable); + if drop_var.as_ref().map(|v| v.to_string()) != Some(arg.name.to_string()) { + Some(convert_if_required( + arg.kind.wildcard(), + &ctx.input, + target_inputset, + var, + )) + } else { + None + } + }) + .collect_vec(); + + let turbofish = self + .signature + .static_defs + .iter() + .map(|def| { + let name = match def { + StaticDefinition::Constant(Argument { name, .. }) => name.to_string(), + StaticDefinition::Generic(name) => name.to_string(), + }; + Expression::Identifier(name.into(), IdentifierType::Symbol) + }) + .collect_vec(); + + let ret_wildcard = ctx + .signature + .return_type + .as_ref() + .and_then(|t| t.wildcard()); + let call = FnCall(Box::new(target_signature.fn_name().into()), args, turbofish).into(); + + self.compose = vec![convert_if_required( + ret_wildcard, + target_inputset, + &ctx.input, + call, + )]; + + Ok(()) + } +} + +impl ToTokens for Intrinsic { + fn to_tokens(&self, tokens: &mut TokenStream) { + let signature = &self.signature; + let fn_name = signature.fn_name().to_string(); + let target_feature = self.target_features.join(","); + let safety = self + .safety + .as_ref() + .expect("safety should be determined during `pre_build`"); + + if let Some(doc) = &self.doc { + let mut doc = vec![doc.to_string()]; + + doc.push(format!("[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/{})", &signature.doc_name())); + + if safety.has_doc_comments() { + doc.push("## Safety".to_string()); + for comment in safety.doc_comments() { + doc.push(format!(" * {comment}")); + } + } else { + assert!( + safety.is_safe(), + "{fn_name} is both public and unsafe, and so needs safety documentation" + ); + } + + tokens.append_all(quote! { #(#[doc = #doc])* }); + } else { + assert!( + matches!(self.visibility, FunctionVisibility::Private), + "{fn_name} needs to be private, or to have documentation." + ); + assert!( + !safety.has_doc_comments(), + "{fn_name} needs a documentation section for its safety comments." + ); + } + + tokens.append_all(quote! { #[inline] }); + + /* If we have manually defined attributes on the block of yaml with + * 'attr:' we want to add them */ + if let Some(attr) = &self.attr { + /* Scan to see if we have defined `FnCall: [target_feature, ['']]`*/ + if !has_target_feature_attr(attr) { + /* If not add the default one that is defined at the top of + * the yaml file. This does mean we scan the attributes vector + * twice, once to see if the `target_feature` exists and again + * to actually append the tokens. We could impose that the + * `target_feature` call has to be the first argument of the + * `attr` block */ + tokens.append_all(quote! { + #[target_feature(enable = #target_feature)] + }); + } + + /* Target feature will get added here */ + let attr_expressions = &mut attr.iter().peekable(); + while let Some(ex) = attr_expressions.next() { + tokens.append(Punct::new('#', Spacing::Alone)); + tokens.append(Punct::new('[', Spacing::Alone)); + ex.to_tokens(tokens); + tokens.append(Punct::new(']', Spacing::Alone)); + } + } else { + tokens.append_all(quote! { + #[target_feature(enable = #target_feature)] + }); + } + + if let Some(assert_instr) = &self.assert_instr { + if !assert_instr.is_empty() { + InstructionAssertionsForBaseType(&assert_instr, &self.base_type.as_ref()) + .to_tokens(tokens) + } + } + + match &self.visibility { + FunctionVisibility::Public => tokens.append_all(quote! { pub }), + FunctionVisibility::Private => {} + } + if safety.is_unsafe() { + tokens.append_all(quote! { unsafe }); + } + tokens.append_all(quote! { #signature }); + tokens.append(Punct::new('{', Spacing::Alone)); + + let mut body_unsafe = false; + let mut expressions = self.compose.iter().peekable(); + while let Some(ex) = expressions.next() { + if !body_unsafe && safety.is_safe() && ex.requires_unsafe_wrapper(&fn_name) { + body_unsafe = true; + tokens.append_all(quote! { unsafe }); + tokens.append(Punct::new('{', Spacing::Alone)); + } + // If it's not the last and not a LLVM link, add a trailing semicolon + if expressions.peek().is_some() && !matches!(ex, Expression::LLVMLink(_)) { + tokens.append_all(quote! { #ex; }) + } else { + ex.to_tokens(tokens) + } + } + if body_unsafe { + tokens.append(Punct::new('}', Spacing::Alone)); + } + + tokens.append(Punct::new('}', Spacing::Alone)); + tokens.append(Punct::new('\n', Spacing::Alone)); + tokens.append(Punct::new('\n', Spacing::Alone)); + } +} + +fn has_target_feature_attr(attrs: &[Expression]) -> bool { + attrs.iter().any(|attr| { + if let Expression::FnCall(fn_call) = attr { + fn_call.is_target_feature_call() + } else { + false + } + }) +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs b/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs new file mode 100644 index 00000000000..d697a8d22d6 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs @@ -0,0 +1,818 @@ +use std::fs::File; +use std::io::Write; +use std::path::PathBuf; +use std::str::FromStr; + +use crate::format_code; +use crate::input::InputType; +use crate::intrinsic::Intrinsic; +use crate::typekinds::BaseType; +use crate::typekinds::{ToRepr, TypeKind}; + +use itertools::Itertools; +use lazy_static::lazy_static; +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; + +// Number of vectors in our buffers - the maximum tuple size, 4, plus 1 as we set the vnum +// argument to 1. +const NUM_VECS: usize = 5; +// The maximum vector length (in bits) +const VL_MAX_BITS: usize = 2048; +// The maximum vector length (in bytes) +const VL_MAX_BYTES: usize = VL_MAX_BITS / 8; +// The maximum number of elements in each vector type +const LEN_F32: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_F64: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_I8: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_I16: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_I32: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_I64: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_U8: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_U16: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_U32: usize = VL_MAX_BYTES / core::mem::size_of::(); +const LEN_U64: usize = VL_MAX_BYTES / core::mem::size_of::(); + +/// `load_intrinsics` and `store_intrinsics` is a vector of intrinsics +/// variants, while `out_path` is a file to write to. +pub fn generate_load_store_tests( + load_intrinsics: Vec, + store_intrinsics: Vec, + out_path: Option<&PathBuf>, +) -> Result<(), String> { + let output = match out_path { + Some(out) => { + Box::new(File::create(out).map_err(|e| format!("couldn't create tests file: {e}"))?) + as Box + } + None => Box::new(std::io::stdout()) as Box, + }; + let mut used_stores = vec![false; store_intrinsics.len()]; + let tests: Vec<_> = load_intrinsics + .iter() + .map(|load| { + let store_candidate = load + .signature + .fn_name() + .to_string() + .replace("svld1s", "svst1") + .replace("svld1u", "svst1") + .replace("svldnt1s", "svstnt1") + .replace("svldnt1u", "svstnt1") + .replace("svld", "svst") + .replace("gather", "scatter"); + + let store_index = store_intrinsics + .iter() + .position(|i| i.signature.fn_name().to_string() == store_candidate); + if let Some(i) = store_index { + used_stores[i] = true; + } + + generate_single_test( + load.clone(), + store_index.map(|i| store_intrinsics[i].clone()), + ) + }) + .try_collect()?; + + assert!(used_stores.into_iter().all(|b| b), "Not all store tests have been paired with a load. Consider generating specifc store-only tests"); + + let preamble = + TokenStream::from_str(&PREAMBLE).map_err(|e| format!("Preamble is invalid: {e}"))?; + // Only output manual tests for the SVE set + let manual_tests = match &load_intrinsics[0].target_features[..] { + [s] if s == "sve" => TokenStream::from_str(&MANUAL_TESTS) + .map_err(|e| format!("Manual tests are invalid: {e}"))?, + _ => quote!(), + }; + format_code( + output, + format!( + "// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen2/spec/sve` and run the following command to re-generate this +// file: +// +// ``` +// cargo run --bin=stdarch-gen2 -- crates/stdarch-gen2/spec +// ``` +{}", + quote! { #preamble #(#tests)* #manual_tests } + ), + ) + .map_err(|e| format!("couldn't write tests: {e}")) +} + +/// A test looks like this: +/// ``` +/// let data = [scalable vector]; +/// +/// let mut storage = [0; N]; +/// +/// store_intrinsic([true_predicate], storage.as_mut_ptr(), data); +/// [test contents of storage] +/// +/// let loaded == load_intrinsic([true_predicate], storage.as_ptr()) +/// assert!(loaded == data); +/// ``` +/// We intialise our data such that the value stored matches the index it's stored to. +/// By doing this we can validate scatters by checking that each value in the storage +/// array is either 0 or the same as its index. +fn generate_single_test( + load: Intrinsic, + store: Option, +) -> Result { + let chars = LdIntrCharacteristics::new(&load)?; + let fn_name = load.signature.fn_name().to_string(); + + if let Some(ty) = &chars.gather_bases_type { + if ty.base_type().unwrap().get_size() == Ok(32) + && chars.gather_index_type.is_none() + && chars.gather_offset_type.is_none() + { + // We lack a way to ensure data is in the bottom 32 bits of the address space + println!("Skipping test for {fn_name}"); + return Ok(quote!()); + } + } + + if fn_name.starts_with("svldff1") && fn_name.contains("gather") { + // TODO: We can remove this check when first-faulting gathers are fixed in CI's QEMU + // https://gitlab.com/qemu-project/qemu/-/issues/1612 + println!("Skipping test for {fn_name}"); + return Ok(quote!()); + } + + let fn_ident = format_ident!("{fn_name}"); + let test_name = format_ident!( + "test_{fn_name}{}", + if let Some(ref store) = store { + format!("_with_{}", store.signature.fn_name()) + } else { + String::new() + } + ); + + let load_type = &chars.load_type; + let acle_type = load_type.acle_notation_repr(); + + // If there's no return type, fallback to the load type for things that depend on it + let ret_type = &load + .signature + .return_type + .as_ref() + .and_then(TypeKind::base_type) + .unwrap_or(load_type); + + let pred_fn = format_ident!("svptrue_b{}", load_type.size()); + + let load_type_caps = load_type.rust_repr().to_uppercase(); + let data_array = format_ident!("{load_type_caps}_DATA"); + + let size_fn = format_ident!("svcnt{}", ret_type.size_literal()); + + let rust_ret_type = ret_type.rust_repr(); + let assert_fn = format_ident!("assert_vector_matches_{rust_ret_type}"); + + // Use vnum=1, so adjust all values by one vector length + let (length_call, vnum_arg) = if chars.vnum { + if chars.is_prf { + (quote!(), quote!(, 1)) + } else { + (quote!(let len = #size_fn() as usize;), quote!(, 1)) + } + } else { + (quote!(), quote!()) + }; + + let (bases_load, bases_arg) = if let Some(ty) = &chars.gather_bases_type { + // Bases is a vector of (sometimes 32-bit) pointers + // When we combine bases with an offset/index argument, we load from the data arrays + // starting at 1 + let base_ty = ty.base_type().unwrap(); + let rust_type = format_ident!("{}", base_ty.rust_repr()); + let index_fn = format_ident!("svindex_{}", base_ty.acle_notation_repr()); + let size_in_bytes = chars.load_type.get_size().unwrap() / 8; + + if base_ty.get_size().unwrap() == 32 { + // Treat bases as a vector of offsets here - we don't test this without an offset or + // index argument + ( + Some(quote!( + let bases = #index_fn(0, #size_in_bytes.try_into().unwrap()); + )), + quote!(, bases), + ) + } else { + // Treat bases as a vector of pointers + let base_fn = format_ident!("svdup_n_{}", base_ty.acle_notation_repr()); + let data_array = if store.is_some() { + format_ident!("storage") + } else { + format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) + }; + + let add_fn = format_ident!("svadd_{}_x", base_ty.acle_notation_repr()); + ( + Some(quote! { + let bases = #base_fn(#data_array.as_ptr() as #rust_type); + let offsets = #index_fn(0, #size_in_bytes.try_into().unwrap()); + let bases = #add_fn(#pred_fn(), bases, offsets); + }), + quote!(, bases), + ) + } + } else { + (None, quote!()) + }; + + let index_arg = if let Some(ty) = &chars.gather_index_type { + let rust_type = format_ident!("{}", ty.rust_repr()); + if chars + .gather_bases_type + .as_ref() + .and_then(TypeKind::base_type) + .map_or(Err(String::new()), BaseType::get_size) + .unwrap() + == 32 + { + // Let index be the base of the data array + let data_array = if store.is_some() { + format_ident!("storage") + } else { + format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) + }; + let size_in_bytes = chars.load_type.get_size().unwrap() / 8; + quote!(, #data_array.as_ptr() as #rust_type / (#size_in_bytes as #rust_type) + 1) + } else { + quote!(, 1.try_into().unwrap()) + } + } else { + quote!() + }; + + let offset_arg = if let Some(ty) = &chars.gather_offset_type { + let size_in_bytes = chars.load_type.get_size().unwrap() / 8; + if chars + .gather_bases_type + .as_ref() + .and_then(TypeKind::base_type) + .map_or(Err(String::new()), BaseType::get_size) + .unwrap() + == 32 + { + // Let offset be the base of the data array + let rust_type = format_ident!("{}", ty.rust_repr()); + let data_array = if store.is_some() { + format_ident!("storage") + } else { + format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) + }; + quote!(, #data_array.as_ptr() as #rust_type + #size_in_bytes as #rust_type) + } else { + quote!(, #size_in_bytes.try_into().unwrap()) + } + } else { + quote!() + }; + + let (offsets_load, offsets_arg) = if let Some(ty) = &chars.gather_offsets_type { + // Offsets is a scalable vector of per-element offsets in bytes. We re-use the contiguous + // data for this, then multiply to get indices + let offsets_fn = format_ident!("svindex_{}", ty.base_type().unwrap().acle_notation_repr()); + let size_in_bytes = chars.load_type.get_size().unwrap() / 8; + ( + Some(quote! { + let offsets = #offsets_fn(0, #size_in_bytes.try_into().unwrap()); + }), + quote!(, offsets), + ) + } else { + (None, quote!()) + }; + + let (indices_load, indices_arg) = if let Some(ty) = &chars.gather_indices_type { + // There's no need to multiply indices by the load type width + let base_ty = ty.base_type().unwrap(); + let indices_fn = format_ident!("svindex_{}", base_ty.acle_notation_repr()); + ( + Some(quote! { + let indices = #indices_fn(0, 1); + }), + quote! {, indices}, + ) + } else { + (None, quote!()) + }; + + let ptr = if chars.gather_bases_type.is_some() { + quote!() + } else if chars.is_prf { + quote!(, I64_DATA.as_ptr()) + } else { + quote!(, #data_array.as_ptr()) + }; + + let tuple_len = &chars.tuple_len; + let expecteds = if chars.is_prf { + // No return value for prefetches + vec![] + } else { + (0..*tuple_len) + .map(|i| get_expected_range(i, &chars)) + .collect() + }; + let asserts: Vec<_> = + if *tuple_len > 1 { + let svget = format_ident!("svget{tuple_len}_{acle_type}"); + expecteds.iter().enumerate().map(|(i, expected)| { + quote! (#assert_fn(#svget::<{ #i as i32 }>(loaded), #expected);) + }).collect() + } else { + expecteds + .iter() + .map(|expected| quote! (#assert_fn(loaded, #expected);)) + .collect() + }; + + let function = if chars.is_prf { + if fn_name.contains("gather") && fn_name.contains("base") && !fn_name.starts_with("svprf_") + { + // svprf(b|h|w|d)_gather base intrinsics do not have a generic type parameter + quote!(#fn_ident::<{ svprfop::SV_PLDL1KEEP }>) + } else { + quote!(#fn_ident::<{ svprfop::SV_PLDL1KEEP }, i64>) + } + } else { + quote!(#fn_ident) + }; + + let octaword_guard = if chars.replicate_width == Some(256) { + let msg = format!("Skipping {test_name} due to SVE vector length"); + quote! { + if svcntb() < 32 { + println!(#msg); + return; + } + } + } else { + quote!() + }; + + let feats = load.target_features.join(","); + + if let Some(store) = store { + let data_init = if *tuple_len == 1 { + quote!(#(#expecteds)*) + } else { + let create = format_ident!("svcreate{tuple_len}_{acle_type}"); + quote!(#create(#(#expecteds),*)) + }; + let input = store.input.types.get(0).unwrap().get(0).unwrap(); + let store_type = input + .get(store.test.get_typeset_index().unwrap()) + .and_then(InputType::typekind) + .and_then(TypeKind::base_type) + .unwrap(); + + let store_type = format_ident!("{}", store_type.rust_repr()); + let storage_len = NUM_VECS * VL_MAX_BITS / chars.load_type.get_size()? as usize; + let store_fn = format_ident!("{}", store.signature.fn_name().to_string()); + let load_type = format_ident!("{}", chars.load_type.rust_repr()); + let (store_ptr, store_mut_ptr) = if chars.gather_bases_type.is_none() { + ( + quote!(, storage.as_ptr() as *const #load_type), + quote!(, storage.as_mut_ptr()), + ) + } else { + (quote!(), quote!()) + }; + let args = quote!(#pred_fn() #store_ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg); + let call = if chars.uses_ffr { + // Doing a normal load first maximises the number of elements our ff/nf test loads + let non_ffr_fn_name = format_ident!( + "{}", + fn_name + .replace("svldff1", "svld1") + .replace("svldnf1", "svld1") + ); + quote! { + svsetffr(); + let _ = #non_ffr_fn_name(#args); + let loaded = #function(#args); + } + } else { + // Note that the FFR must be set for all tests as the assert functions mask against it + quote! { + svsetffr(); + let loaded = #function(#args); + } + }; + + Ok(quote! { + #[simd_test(enable = #feats)] + unsafe fn #test_name() { + #octaword_guard + #length_call + let mut storage = [0 as #store_type; #storage_len]; + let data = #data_init; + #bases_load + #offsets_load + #indices_load + + #store_fn(#pred_fn() #store_mut_ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg, data); + for (i, &val) in storage.iter().enumerate() { + assert!(val == 0 as #store_type || val == i as #store_type); + } + + #call + #(#asserts)* + + } + }) + } else { + let args = quote!(#pred_fn() #ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg); + let call = if chars.uses_ffr { + // Doing a normal load first maximises the number of elements our ff/nf test loads + let non_ffr_fn_name = format_ident!( + "{}", + fn_name + .replace("svldff1", "svld1") + .replace("svldnf1", "svld1") + ); + quote! { + svsetffr(); + let _ = #non_ffr_fn_name(#args); + let loaded = #function(#args); + } + } else { + // Note that the FFR must be set for all tests as the assert functions mask against it + quote! { + svsetffr(); + let loaded = #function(#args); + } + }; + Ok(quote! { + #[simd_test(enable = #feats)] + unsafe fn #test_name() { + #octaword_guard + #bases_load + #offsets_load + #indices_load + #call + #length_call + + #(#asserts)* + } + }) + } +} + +/// Assumes chars.ret_type is not None +fn get_expected_range(tuple_idx: usize, chars: &LdIntrCharacteristics) -> proc_macro2::TokenStream { + // vnum=1 + let vnum_adjust = if chars.vnum { quote!(len+) } else { quote!() }; + + let bases_adjust = + (chars.gather_index_type.is_some() || chars.gather_offset_type.is_some()) as usize; + + let tuple_len = chars.tuple_len; + let size = chars + .ret_type + .as_ref() + .and_then(TypeKind::base_type) + .unwrap_or(&chars.load_type) + .get_size() + .unwrap() as usize; + + if chars.replicate_width == Some(128) { + // svld1rq + let ty_rust = format_ident!( + "{}", + chars + .ret_type + .as_ref() + .unwrap() + .base_type() + .unwrap() + .rust_repr() + ); + let args: Vec<_> = (0..(128 / size)).map(|i| quote!(#i as #ty_rust)).collect(); + let dup = format_ident!( + "svdupq_n_{}", + chars.ret_type.as_ref().unwrap().acle_notation_repr() + ); + quote!(#dup(#(#args,)*)) + } else if chars.replicate_width == Some(256) { + // svld1ro - we use two interleaved svdups to create a repeating 256-bit pattern + let ty_rust = format_ident!( + "{}", + chars + .ret_type + .as_ref() + .unwrap() + .base_type() + .unwrap() + .rust_repr() + ); + let ret_acle = chars.ret_type.as_ref().unwrap().acle_notation_repr(); + let args: Vec<_> = (0..(128 / size)).map(|i| quote!(#i as #ty_rust)).collect(); + let args2: Vec<_> = ((128 / size)..(256 / size)) + .map(|i| quote!(#i as #ty_rust)) + .collect(); + let dup = format_ident!("svdupq_n_{ret_acle}"); + let interleave = format_ident!("svtrn1q_{ret_acle}"); + quote!(#interleave(#dup(#(#args,)*), #dup(#(#args2,)*))) + } else { + let start = bases_adjust + tuple_idx; + if chars + .ret_type + .as_ref() + .unwrap() + .base_type() + .unwrap() + .is_float() + { + // Use svcvt to create a linear sequence of floats + let cvt_fn = format_ident!("svcvt_f{size}_s{size}_x"); + let pred_fn = format_ident!("svptrue_b{size}"); + let svindex_fn = format_ident!("svindex_s{size}"); + quote! { #cvt_fn(#pred_fn(), #svindex_fn((#vnum_adjust #start).try_into().unwrap(), #tuple_len.try_into().unwrap()))} + } else { + let ret_acle = chars.ret_type.as_ref().unwrap().acle_notation_repr(); + let svindex = format_ident!("svindex_{ret_acle}"); + quote!(#svindex((#vnum_adjust #start).try_into().unwrap(), #tuple_len.try_into().unwrap())) + } + } +} + +struct LdIntrCharacteristics { + // The data type to load from (not necessarily the data type returned) + load_type: BaseType, + // The data type to return (None for unit) + ret_type: Option, + // The size of tuple to load/store + tuple_len: usize, + // Whether a vnum argument is present + vnum: bool, + // Is the intrinsic first/non-faulting? + uses_ffr: bool, + // Is it a prefetch? + is_prf: bool, + // The size of data loaded with svld1ro/q intrinsics + replicate_width: Option, + // Scalable vector of pointers to load from + gather_bases_type: Option, + // Scalar offset, paired with bases + gather_offset_type: Option, + // Scalar index, paired with bases + gather_index_type: Option, + // Scalable vector of offsets + gather_offsets_type: Option, + // Scalable vector of indices + gather_indices_type: Option, +} + +impl LdIntrCharacteristics { + fn new(intr: &Intrinsic) -> Result { + let input = intr.input.types.get(0).unwrap().get(0).unwrap(); + let load_type = input + .get(intr.test.get_typeset_index().unwrap()) + .and_then(InputType::typekind) + .and_then(TypeKind::base_type) + .unwrap(); + + let ret_type = intr.signature.return_type.clone(); + + let name = intr.signature.fn_name().to_string(); + let tuple_len = name + .chars() + .find(|c| c.is_numeric()) + .and_then(|c| c.to_digit(10)) + .unwrap_or(1) as usize; + + let uses_ffr = name.starts_with("svldff") || name.starts_with("svldnf"); + + let is_prf = name.starts_with("svprf"); + + let replicate_width = if name.starts_with("svld1ro") { + Some(256) + } else if name.starts_with("svld1rq") { + Some(128) + } else { + None + }; + + let get_ty_of_arg = |name: &str| { + intr.signature + .arguments + .iter() + .find(|a| a.name.to_string() == name) + .map(|a| a.kind.clone()) + }; + + let gather_bases_type = get_ty_of_arg("bases"); + let gather_offset_type = get_ty_of_arg("offset"); + let gather_index_type = get_ty_of_arg("index"); + let gather_offsets_type = get_ty_of_arg("offsets"); + let gather_indices_type = get_ty_of_arg("indices"); + + Ok(LdIntrCharacteristics { + load_type: *load_type, + ret_type, + tuple_len, + vnum: name.contains("vnum"), + uses_ffr, + is_prf, + replicate_width, + gather_bases_type, + gather_offset_type, + gather_index_type, + gather_offsets_type, + gather_indices_type, + }) + } +} + +lazy_static! { + static ref PREAMBLE: String = format!( + r#"#![allow(unused)] + +use super::*; +use std::boxed::Box; +use std::convert::{{TryFrom, TryInto}}; +use std::sync::LazyLock; +use std::vec::Vec; +use stdarch_test::simd_test; + +static F32_DATA: LazyLock<[f32; {LEN_F32} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_F32} * {NUM_VECS}) + .map(|i| i as f32) + .collect::>() + .try_into() + .expect("f32 data incorrectly initialised") +}}); +static F64_DATA: LazyLock<[f64; {LEN_F64} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_F64} * {NUM_VECS}) + .map(|i| i as f64) + .collect::>() + .try_into() + .expect("f64 data incorrectly initialised") +}}); +static I8_DATA: LazyLock<[i8; {LEN_I8} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_I8} * {NUM_VECS}) + .map(|i| ((i + 128) % 256 - 128) as i8) + .collect::>() + .try_into() + .expect("i8 data incorrectly initialised") +}}); +static I16_DATA: LazyLock<[i16; {LEN_I16} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_I16} * {NUM_VECS}) + .map(|i| i as i16) + .collect::>() + .try_into() + .expect("i16 data incorrectly initialised") +}}); +static I32_DATA: LazyLock<[i32; {LEN_I32} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_I32} * {NUM_VECS}) + .map(|i| i as i32) + .collect::>() + .try_into() + .expect("i32 data incorrectly initialised") +}}); +static I64_DATA: LazyLock<[i64; {LEN_I64} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_I64} * {NUM_VECS}) + .map(|i| i as i64) + .collect::>() + .try_into() + .expect("i64 data incorrectly initialised") +}}); +static U8_DATA: LazyLock<[u8; {LEN_U8} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_U8} * {NUM_VECS}) + .map(|i| i as u8) + .collect::>() + .try_into() + .expect("u8 data incorrectly initialised") +}}); +static U16_DATA: LazyLock<[u16; {LEN_U16} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_U16} * {NUM_VECS}) + .map(|i| i as u16) + .collect::>() + .try_into() + .expect("u16 data incorrectly initialised") +}}); +static U32_DATA: LazyLock<[u32; {LEN_U32} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_U32} * {NUM_VECS}) + .map(|i| i as u32) + .collect::>() + .try_into() + .expect("u32 data incorrectly initialised") +}}); +static U64_DATA: LazyLock<[u64; {LEN_U64} * {NUM_VECS}]> = LazyLock::new(|| {{ + (0..{LEN_U64} * {NUM_VECS}) + .map(|i| i as u64) + .collect::>() + .try_into() + .expect("u64 data incorrectly initialised") +}}); + +#[target_feature(enable = "sve")] +fn assert_vector_matches_f32(vector: svfloat32_t, expected: svfloat32_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b32(), defined)); + let cmp = svcmpne_f32(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_f64(vector: svfloat64_t, expected: svfloat64_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b64(), defined)); + let cmp = svcmpne_f64(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_i8(vector: svint8_t, expected: svint8_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b8(), defined)); + let cmp = svcmpne_s8(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_i16(vector: svint16_t, expected: svint16_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b16(), defined)); + let cmp = svcmpne_s16(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_i32(vector: svint32_t, expected: svint32_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b32(), defined)); + let cmp = svcmpne_s32(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_i64(vector: svint64_t, expected: svint64_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b64(), defined)); + let cmp = svcmpne_s64(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_u8(vector: svuint8_t, expected: svuint8_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b8(), defined)); + let cmp = svcmpne_u8(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_u16(vector: svuint16_t, expected: svuint16_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b16(), defined)); + let cmp = svcmpne_u16(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_u32(vector: svuint32_t, expected: svuint32_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b32(), defined)); + let cmp = svcmpne_u32(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} + +#[target_feature(enable = "sve")] +fn assert_vector_matches_u64(vector: svuint64_t, expected: svuint64_t) {{ + let defined = svrdffr(); + assert!(svptest_first(svptrue_b64(), defined)); + let cmp = svcmpne_u64(defined, vector, expected); + assert!(!svptest_any(defined, cmp)) +}} +"# + ); +} + +lazy_static! { + static ref MANUAL_TESTS: String = format!( + "#[simd_test(enable = \"sve\")] +unsafe fn test_ffr() {{ + svsetffr(); + let ffr = svrdffr(); + assert_vector_matches_u8(svdup_n_u8_z(ffr, 1), svindex_u8(1, 0)); + let pred = svdupq_n_b8(true, false, true, false, true, false, true, false, + true, false, true, false, true, false, true, false); + svwrffr(pred); + let ffr = svrdffr_z(svptrue_b8()); + assert_vector_matches_u8(svdup_n_u8_z(ffr, 1), svdup_n_u8_z(pred, 1)); +}} +" + ); +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/main.rs b/library/stdarch/crates/stdarch-gen-arm/src/main.rs new file mode 100644 index 00000000000..22bf6724b02 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/main.rs @@ -0,0 +1,279 @@ +#![feature(pattern)] + +mod assert_instr; +mod context; +mod expression; +mod fn_suffix; +mod input; +mod intrinsic; +mod load_store_tests; +mod matching; +mod predicate_forms; +mod typekinds; +mod wildcards; +mod wildstring; + +use intrinsic::Test; +use itertools::Itertools; +use quote::quote; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use walkdir::WalkDir; + +fn main() -> Result<(), String> { + parse_args() + .into_iter() + .map(|(filepath, out)| { + File::open(&filepath) + .map(|f| (f, filepath, out)) + .map_err(|e| format!("could not read input file: {e}")) + }) + .map(|res| { + let (file, filepath, out) = res?; + serde_yaml::from_reader(file) + .map(|input: input::GeneratorInput| (input, filepath, out)) + .map_err(|e| format!("could not parse input file: {e}")) + }) + .collect::, _>>()? + .into_iter() + .map(|(input, filepath, out)| { + let intrinsics = input.intrinsics.into_iter() + .map(|intrinsic| { + intrinsic.generate_variants(&input.ctx) + }) + .try_collect() + .map(|mut vv: Vec<_>| { + vv.sort_by_cached_key(|variants| { + variants.first().map_or_else(String::default, |variant| { + variant.signature.fn_name().to_string() + }) + }); + vv.into_iter().flatten().collect_vec() + })?; + + if filepath.ends_with("sve.spec.yml") || filepath.ends_with("sve2.spec.yml") { + let loads = intrinsics.iter() + .filter_map(|i| { + if matches!(i.test, Test::Load(..)) { + Some(i.clone()) + } else { + None + } + }).collect(); + let stores = intrinsics.iter() + .filter_map(|i| { + if matches!(i.test, Test::Store(..)) { + Some(i.clone()) + } else { + None + } + }).collect(); + load_store_tests::generate_load_store_tests(loads, stores, out.as_ref().map(|o| make_tests_filepath(&filepath, o)).as_ref())?; + } + + Ok(( + input::GeneratorInput { + intrinsics, + ctx: input.ctx, + }, + filepath, + out, + )) + }) + .try_for_each( + |result: context::Result<(input::GeneratorInput, PathBuf, Option)>| -> context::Result { + let (generated, filepath, out) = result?; + + let w = match out { + Some(out) => Box::new( + File::create(make_output_filepath(&filepath, &out)) + .map_err(|e| format!("could not create output file: {e}"))?, + ) as Box, + None => Box::new(std::io::stdout()) as Box, + }; + + generate_file(generated, w) + .map_err(|e| format!("could not generate output file: {e}")) + }, + ) +} + +fn parse_args() -> Vec<(PathBuf, Option)> { + let mut args_it = std::env::args().skip(1); + assert!( + 1 <= args_it.len() && args_it.len() <= 2, + "Usage: cargo run -p stdarch-gen2 -- INPUT_DIR [OUTPUT_DIR]" + ); + + let in_path = Path::new(args_it.next().unwrap().as_str()).to_path_buf(); + assert!( + in_path.exists() && in_path.is_dir(), + "invalid path {in_path:#?} given" + ); + + let out_dir = if let Some(dir) = args_it.next() { + let out_path = Path::new(dir.as_str()).to_path_buf(); + assert!( + out_path.exists() && out_path.is_dir(), + "invalid path {out_path:#?} given" + ); + Some(out_path) + } else { + std::env::current_exe() + .map(|mut f| { + f.pop(); + f.push("../../crates/core_arch/src/aarch64/"); + f.exists().then_some(f) + }) + .ok() + .flatten() + }; + + WalkDir::new(in_path) + .into_iter() + .filter_map(Result::ok) + .filter(|f| f.file_type().is_file()) + .map(|f| (f.into_path(), out_dir.clone())) + .collect() +} + +fn generate_file( + generated_input: input::GeneratorInput, + mut out: Box, +) -> std::io::Result<()> { + write!( + out, + r#"// This code is automatically generated. DO NOT MODIFY. +// +// Instead, modify `crates/stdarch-gen2/spec/` and run the following command to re-generate this file: +// +// ``` +// cargo run --bin=stdarch-gen2 -- crates/stdarch-gen2/spec +// ``` +#![allow(improper_ctypes)] + +#[cfg(test)] +use stdarch_test::assert_instr; + +use super::*;{uses_neon} + +"#, + uses_neon = generated_input + .ctx + .uses_neon_types + .then_some("\nuse crate::core_arch::arch::aarch64::*;") + .unwrap_or_default(), + )?; + let intrinsics = generated_input.intrinsics; + format_code(out, quote! { #(#intrinsics)* })?; + Ok(()) +} + +pub fn format_code( + mut output: impl std::io::Write, + input: impl std::fmt::Display, +) -> std::io::Result<()> { + let proc = Command::new("rustfmt") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?; + write!(proc.stdin.as_ref().unwrap(), "{input}")?; + output.write_all(proc.wait_with_output()?.stdout.as_slice()) +} + +/// Derive an output file name from an input file and an output directory. +/// +/// The name is formed by: +/// +/// - ... taking in_filepath.file_name() (dropping all directory components), +/// - ... dropping a .yml or .yaml extension (if present), +/// - ... then dropping a .spec extension (if present). +/// +/// Panics if the resulting name is empty, or if file_name() is not UTF-8. +fn make_output_filepath(in_filepath: &Path, out_dirpath: &Path) -> PathBuf { + make_filepath(in_filepath, out_dirpath, |name: &str| format!("{name}.rs")) +} + +fn make_tests_filepath(in_filepath: &Path, out_dirpath: &Path) -> PathBuf { + make_filepath(in_filepath, out_dirpath, |name: &str| { + format!("ld_st_tests_{name}.rs") + }) +} + +fn make_filepath String>( + in_filepath: &Path, + out_dirpath: &Path, + name_formatter: F, +) -> PathBuf { + let mut parts = in_filepath.iter(); + let name = parts + .next_back() + .and_then(|f| f.to_str()) + .expect("Inputs must have valid, UTF-8 file_name()"); + let dir = parts.next_back().unwrap(); + + let name = name + .trim_end_matches(".yml") + .trim_end_matches(".yaml") + .trim_end_matches(".spec"); + assert!(!name.is_empty()); + + let mut output = out_dirpath.to_path_buf(); + output.push(dir); + output.push(name_formatter(name)); + output +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn infer_output_file() { + macro_rules! t { + ($src:expr, $outdir:expr, $dst:expr) => { + let src: PathBuf = $src.iter().collect(); + let outdir: PathBuf = $outdir.iter().collect(); + let dst: PathBuf = $dst.iter().collect(); + assert_eq!(make_output_filepath(&src, &outdir), dst); + }; + } + // Documented usage. + t!(["x", "NAME.spec.yml"], [""], ["x", "NAME.rs"]); + t!( + ["x", "NAME.spec.yml"], + ["a", "b"], + ["a", "b", "x", "NAME.rs"] + ); + t!( + ["x", "y", "NAME.spec.yml"], + ["out"], + ["out", "y", "NAME.rs"] + ); + t!(["x", "NAME.spec.yaml"], ["out"], ["out", "x", "NAME.rs"]); + t!(["x", "NAME.spec"], ["out"], ["out", "x", "NAME.rs"]); + t!(["x", "NAME.yml"], ["out"], ["out", "x", "NAME.rs"]); + t!(["x", "NAME.yaml"], ["out"], ["out", "x", "NAME.rs"]); + // Unrecognised extensions get treated as part of the stem. + t!( + ["x", "NAME.spac.yml"], + ["out"], + ["out", "x", "NAME.spac.rs"] + ); + t!(["x", "NAME.txt"], ["out"], ["out", "x", "NAME.txt.rs"]); + // Always take the top-level directory from the input path + t!( + ["x", "y", "z", "NAME.spec.yml"], + ["out"], + ["out", "z", "NAME.rs"] + ); + } + + #[test] + #[should_panic] + fn infer_output_file_no_stem() { + make_output_filepath(Path::new(".spec.yml"), Path::new("")); + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/matching.rs b/library/stdarch/crates/stdarch-gen-arm/src/matching.rs new file mode 100644 index 00000000000..0c480620428 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/matching.rs @@ -0,0 +1,170 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; +use serde::{Deserialize, Serialize}; +use std::fmt; + +use crate::context::{self, LocalContext}; +use crate::typekinds::{BaseType, BaseTypeKind, TypeKind}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MatchSizeValues { + pub default: T, + pub byte: Option, + pub halfword: Option, + pub doubleword: Option, +} + +impl MatchSizeValues { + pub fn get(&mut self, ty: &TypeKind, ctx: &LocalContext) -> context::Result<&T> { + let base_ty = if let Some(w) = ty.wildcard() { + ctx.provide_type_wildcard(w)? + } else { + ty.clone() + }; + + if let BaseType::Sized(_, bitsize) = base_ty.base_type().unwrap() { + match (bitsize, &self.byte, &self.halfword, &self.doubleword) { + (64, _, _, Some(v)) | (16, _, Some(v), _) | (8, Some(v), _, _) => Ok(v), + _ => Ok(&self.default), + } + } else { + Err(format!("cannot match bitsize to unsized type {ty:?}!")) + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MatchKindValues { + pub default: T, + pub float: Option, + pub unsigned: Option, +} + +impl MatchKindValues { + pub fn get(&mut self, ty: &TypeKind, ctx: &LocalContext) -> context::Result<&T> { + let base_ty = if let Some(w) = ty.wildcard() { + ctx.provide_type_wildcard(w)? + } else { + ty.clone() + }; + + match ( + base_ty.base_type().unwrap().kind(), + &self.float, + &self.unsigned, + ) { + (BaseTypeKind::Float, Some(v), _) | (BaseTypeKind::UInt, _, Some(v)) => Ok(v), + _ => Ok(&self.default), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum SizeMatchable { + Matched(T), + Unmatched { + match_size: Option, + #[serde(flatten)] + values: MatchSizeValues>, + }, +} + +impl SizeMatchable { + pub fn perform_match(&mut self, ctx: &LocalContext) -> context::Result { + match self { + Self::Unmatched { + match_size: None, + values: MatchSizeValues { default, .. }, + } => *self = Self::Matched(*default.to_owned()), + Self::Unmatched { + match_size: Some(ty), + values, + } => *self = Self::Matched(*values.get(ty, ctx)?.to_owned()), + _ => {} + } + Ok(()) + } +} + +impl AsRef for SizeMatchable { + fn as_ref(&self) -> &T { + if let SizeMatchable::Matched(v) = self { + v + } else { + panic!("no match for {self:?} was performed"); + } + } +} + +impl AsMut for SizeMatchable { + fn as_mut(&mut self) -> &mut T { + if let SizeMatchable::Matched(v) = self { + v + } else { + panic!("no match for {self:?} was performed"); + } + } +} + +impl ToTokens for SizeMatchable { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.as_ref().to_tokens(tokens) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum KindMatchable { + Matched(T), + Unmatched { + match_kind: Option, + #[serde(flatten)] + values: MatchKindValues>, + }, +} + +impl KindMatchable { + pub fn perform_match(&mut self, ctx: &LocalContext) -> context::Result { + match self { + Self::Unmatched { + match_kind: None, + values: MatchKindValues { default, .. }, + } => *self = Self::Matched(*default.to_owned()), + Self::Unmatched { + match_kind: Some(ty), + values, + } => *self = Self::Matched(*values.get(ty, ctx)?.to_owned()), + _ => {} + } + Ok(()) + } +} + +impl AsRef for KindMatchable { + fn as_ref(&self) -> &T { + if let KindMatchable::Matched(v) = self { + v + } else { + panic!("no match for {self:?} was performed"); + } + } +} + +impl AsMut for KindMatchable { + fn as_mut(&mut self) -> &mut T { + if let KindMatchable::Matched(v) = self { + v + } else { + panic!("no match for {self:?} was performed"); + } + } +} + +impl ToTokens for KindMatchable { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.as_ref().to_tokens(tokens) + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/predicate_forms.rs b/library/stdarch/crates/stdarch-gen-arm/src/predicate_forms.rs new file mode 100644 index 00000000000..02789bf7eb0 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/predicate_forms.rs @@ -0,0 +1,249 @@ +use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use std::fmt; +use std::str::FromStr; + +use crate::context; +use crate::expression::{Expression, FnCall, IdentifierType}; +use crate::intrinsic::Intrinsic; +use crate::typekinds::{ToRepr, TypeKind}; +use crate::wildcards::Wildcard; +use crate::wildstring::WildString; + +const ZEROING_SUFFIX: &str = "_z"; +const MERGING_SUFFIX: &str = "_m"; +const DONT_CARE_SUFFIX: &str = "_x"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ZeroingMethod { + /// Drop the specified argument and replace it with a zeroinitializer + Drop { drop: WildString }, + /// Apply zero selection to the specified variable when zeroing + Select { select: WildString }, +} + +impl PartialOrd for ZeroingMethod { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ZeroingMethod { + fn cmp(&self, _: &Self) -> std::cmp::Ordering { + std::cmp::Ordering::Equal + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum DontCareMethod { + #[default] + Inferred, + AsZeroing, + AsMerging, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct PredicationMethods { + /// Zeroing method, if the zeroing predicate form is used + #[serde(default)] + pub zeroing_method: Option, + /// Don't care method, if the don't care predicate form is used + #[serde(default)] + pub dont_care_method: DontCareMethod, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub enum PredicateForm { + /// Enables merging predicate form + Merging, + /// Enables "don't care" predicate form. + DontCare(DontCareMethod), + /// Enables zeroing predicate form. If LLVM zeroselection is performed, then + /// set the `select` field to the variable that gets set. Otherwise set the + /// `drop` field if the zeroinitializer replaces a predicate when merging. + Zeroing(ZeroingMethod), +} + +impl PredicateForm { + pub fn get_suffix(&self) -> &'static str { + match self { + PredicateForm::Zeroing { .. } => ZEROING_SUFFIX, + PredicateForm::Merging => MERGING_SUFFIX, + PredicateForm::DontCare { .. } => DONT_CARE_SUFFIX, + } + } + + pub fn make_zeroinitializer(ty: &TypeKind) -> Expression { + FnCall::new_expression( + format!("svdup_n_{}", ty.acle_notation_repr()) + .parse() + .unwrap(), + vec![if ty.base_type().unwrap().is_float() { + Expression::FloatConstant(0.0) + } else { + Expression::IntConstant(0) + }], + ) + } + + pub fn make_zeroselector(pg_var: WildString, op_var: WildString, ty: &TypeKind) -> Expression { + FnCall::new_expression( + format!("svsel_{}", ty.acle_notation_repr()) + .parse() + .unwrap(), + vec![ + Expression::Identifier(pg_var, IdentifierType::Variable), + Expression::Identifier(op_var, IdentifierType::Variable), + Self::make_zeroinitializer(ty), + ], + ) + } + + pub fn post_build(&self, intrinsic: &mut Intrinsic) -> context::Result { + // Drop the argument + match self { + PredicateForm::Zeroing(ZeroingMethod::Drop { drop: drop_var }) => { + intrinsic.signature.drop_argument(drop_var)? + } + PredicateForm::DontCare(DontCareMethod::AsZeroing) => { + if let ZeroingMethod::Drop { drop } = intrinsic + .input + .predication_methods + .zeroing_method + .to_owned() + .ok_or_else(|| { + "DontCareMethod::AsZeroing without zeroing method.".to_string() + })? + { + intrinsic.signature.drop_argument(&drop)? + } + } + _ => {} + } + + Ok(()) + } + + fn infer_dont_care(mask: &PredicationMask, methods: &PredicationMethods) -> PredicateForm { + let method = if methods.dont_care_method == DontCareMethod::Inferred { + if mask.has_zeroing() + && matches!(methods.zeroing_method, Some(ZeroingMethod::Drop { .. })) + { + DontCareMethod::AsZeroing + } else { + DontCareMethod::AsMerging + } + } else { + methods.dont_care_method + }; + + PredicateForm::DontCare(method) + } + + pub fn compile_list( + mask: &PredicationMask, + methods: &PredicationMethods, + ) -> context::Result> { + let mut forms = Vec::new(); + + if mask.has_merging() { + forms.push(PredicateForm::Merging) + } + + if mask.has_dont_care() { + forms.push(Self::infer_dont_care(mask, methods)) + } + + if mask.has_zeroing() { + if let Some(method) = methods.zeroing_method.to_owned() { + forms.push(PredicateForm::Zeroing(method)) + } else { + return Err( + "cannot create a zeroing variant without a zeroing method specified!" + .to_string(), + ); + } + } + + Ok(forms) + } +} + +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, Hash, DeserializeFromStr, SerializeDisplay, +)] +pub struct PredicationMask { + /// Merging + m: bool, + /// Don't care + x: bool, + /// Zeroing + z: bool, +} + +impl PredicationMask { + pub fn has_merging(&self) -> bool { + self.m + } + + pub fn has_dont_care(&self) -> bool { + self.x + } + + pub fn has_zeroing(&self) -> bool { + self.z + } +} + +impl FromStr for PredicationMask { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut result = Self::default(); + for kind in s.bytes() { + match kind { + b'm' => result.m = true, + b'x' => result.x = true, + b'z' => result.z = true, + _ => { + return Err(format!( + "unknown predicate form modifier: {}", + char::from(kind) + )); + } + } + } + + if result.m || result.x || result.z { + Ok(result) + } else { + Err("invalid predication mask".to_string()) + } + } +} + +impl fmt::Display for PredicationMask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.m.then(|| write!(f, "m")).transpose()?; + self.x.then(|| write!(f, "x")).transpose()?; + self.z.then(|| write!(f, "z")).transpose().map(|_| ()) + } +} + +impl TryFrom<&WildString> for PredicationMask { + type Error = String; + + fn try_from(value: &WildString) -> Result { + value + .wildcards() + .find_map(|w| { + if let Wildcard::PredicateForms(mask) = w { + Some(*mask) + } else { + None + } + }) + .ok_or_else(|| "no predicate forms were specified in the name".to_string()) + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/typekinds.rs b/library/stdarch/crates/stdarch-gen-arm/src/typekinds.rs new file mode 100644 index 00000000000..7a4fed85ce7 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/typekinds.rs @@ -0,0 +1,1051 @@ +use lazy_static::lazy_static; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens, TokenStreamExt}; +use regex::Regex; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use std::fmt; +use std::str::FromStr; + +use crate::context; +use crate::expression::{Expression, FnCall}; +use crate::intrinsic::AccessLevel; +use crate::wildcards::Wildcard; + +const VECTOR_FULL_REGISTER_SIZE: u32 = 128; +const VECTOR_HALF_REGISTER_SIZE: u32 = VECTOR_FULL_REGISTER_SIZE / 2; + +#[derive(Debug, Clone, Copy)] +pub enum TypeRepr { + C, + Rust, + LLVMMachine, + ACLENotation, + Size, + SizeLiteral, + TypeKind, + SizeInBytesLog2, +} + +pub trait ToRepr { + fn repr(&self, repr: TypeRepr) -> String; + + fn c_repr(&self) -> String { + self.repr(TypeRepr::C) + } + + fn rust_repr(&self) -> String { + self.repr(TypeRepr::Rust) + } + + fn llvm_machine_repr(&self) -> String { + self.repr(TypeRepr::LLVMMachine) + } + + fn acle_notation_repr(&self) -> String { + self.repr(TypeRepr::ACLENotation) + } + + fn size(&self) -> String { + self.repr(TypeRepr::Size) + } + + fn size_literal(&self) -> String { + self.repr(TypeRepr::SizeLiteral) + } + + fn type_kind(&self) -> String { + self.repr(TypeRepr::TypeKind) + } + + fn size_in_bytes_log2(&self) -> String { + self.repr(TypeRepr::SizeInBytesLog2) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] +pub struct TypeKindOptions { + f: bool, + s: bool, + u: bool, + p: bool, +} + +impl TypeKindOptions { + pub fn contains(&self, kind: BaseTypeKind) -> bool { + match kind { + BaseTypeKind::Float => self.f, + BaseTypeKind::Int => self.s, + BaseTypeKind::UInt => self.u, + BaseTypeKind::Poly => self.p, + BaseTypeKind::Bool => false, + } + } +} + +impl FromStr for TypeKindOptions { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut result = Self::default(); + for kind in s.bytes() { + match kind { + b'f' => result.f = true, + b's' => result.s = true, + b'u' => result.u = true, + b'p' => result.p = true, + _ => { + return Err(format!("unknown type kind: {}", char::from(kind))); + } + } + } + Ok(result) + } +} + +impl fmt::Display for TypeKindOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.f.then(|| write!(f, "f")).transpose()?; + self.s.then(|| write!(f, "s")).transpose()?; + self.u.then(|| write!(f, "u")).transpose().map(|_| ()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BaseTypeKind { + Float, + Int, + UInt, + Bool, + Poly, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BaseType { + Sized(BaseTypeKind, u32), + Unsized(BaseTypeKind), +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, SerializeDisplay, DeserializeFromStr, +)] +pub enum VectorTupleSize { + Two, + Three, + Four, +} + +impl VectorTupleSize { + pub fn to_int(&self) -> u32 { + match self { + Self::Two => 2, + Self::Three => 3, + Self::Four => 4, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VectorType { + base_type: BaseType, + lanes: u32, + is_scalable: bool, + tuple_size: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] +pub enum TypeKind { + Vector(VectorType), + Base(BaseType), + Pointer(Box, AccessLevel), + Custom(String), + Wildcard(Wildcard), +} + +impl TypeKind { + pub fn base_type(&self) -> Option<&BaseType> { + match self { + Self::Vector(t) => Some(t.base_type()), + Self::Pointer(t, _) => t.base_type(), + Self::Base(t) => Some(t), + Self::Wildcard(..) => None, + Self::Custom(..) => None, + } + } + + pub fn base_type_mut(&mut self) -> Option<&mut BaseType> { + match self { + Self::Vector(t) => Some(t.base_type_mut()), + Self::Pointer(t, _) => t.base_type_mut(), + Self::Base(t) => Some(t), + Self::Wildcard(..) => None, + Self::Custom(..) => None, + } + } + + pub fn populate_wildcard(&mut self, type_kind: TypeKind) -> context::Result { + match self { + Self::Wildcard(..) => *self = type_kind, + Self::Pointer(t, _) => t.populate_wildcard(type_kind)?, + _ => return Err("no wildcard available to populate".to_string()), + } + Ok(()) + } + + pub fn base(&self) -> Option<&BaseType> { + match self { + Self::Base(ty) => Some(ty), + Self::Pointer(tk, _) => tk.base(), + Self::Vector(ty) => Some(&ty.base_type), + _ => None, + } + } + + pub fn vector(&self) -> Option<&VectorType> { + match self { + Self::Vector(ty) => Some(ty), + _ => None, + } + } + + pub fn vector_mut(&mut self) -> Option<&mut VectorType> { + match self { + Self::Vector(ty) => Some(ty), + _ => None, + } + } + + pub fn wildcard(&self) -> Option<&Wildcard> { + match self { + Self::Wildcard(w) => Some(w), + Self::Pointer(w, _) => w.wildcard(), + _ => None, + } + } + + pub fn make_predicate_from(ty: &TypeKind) -> context::Result { + Ok(TypeKind::Vector(VectorType::make_predicate_from_bitsize( + ty.base_type() + .ok_or_else(|| format!("cannot infer predicate from type {ty}"))? + .get_size() + .map_err(|_| format!("cannot infer predicate from unsized type {ty}"))?, + ))) + } + + pub fn make_vector( + from: TypeKind, + is_scalable: bool, + tuple_size: Option, + ) -> context::Result { + from.base().cloned().map_or_else( + || Err(format!("cannot make a vector type out of {from}!")), + |base| { + let vt = VectorType::make_from_base(base, is_scalable, tuple_size); + Ok(TypeKind::Vector(vt)) + }, + ) + } + + /// Return a new expression that converts the provided `expr` from type `other` to `self`. + /// + /// Conversions are bitwise over the whole value, like `transmute`, though `transmute` + /// itself is only used as a last resort. + /// + /// This can fail (returning `None`) due to incompatible types, and many conversions are simply + /// unimplemented. + pub fn express_reinterpretation_from( + &self, + other: &TypeKind, + expr: impl Into, + ) -> Option { + if self == other { + Some(expr.into()) + } else if let (Some(self_vty), Some(other_vty)) = (self.vector(), other.vector()) { + if self_vty.is_scalable + && self_vty.tuple_size.is_none() + && other_vty.is_scalable + && other_vty.tuple_size.is_none() + { + // Plain scalable vectors. + use BaseTypeKind::*; + match (self_vty.base_type, other_vty.base_type) { + (BaseType::Sized(Int, self_size), BaseType::Sized(UInt, other_size)) + if self_size == other_size => + { + Some(Expression::MethodCall( + Box::new(expr.into()), + "as_signed".parse().unwrap(), + vec![], + )) + } + (BaseType::Sized(UInt, self_size), BaseType::Sized(Int, other_size)) + if self_size == other_size => + { + Some(Expression::MethodCall( + Box::new(expr.into()), + "as_unsigned".parse().unwrap(), + vec![], + )) + } + ( + BaseType::Sized(Float | Int | UInt, _), + BaseType::Sized(Float | Int | UInt, _), + ) => Some(FnCall::new_expression( + // Conversions between float and (u)int, or where the lane size changes. + "simd_reinterpret".parse().unwrap(), + vec![expr.into()], + )), + _ => None, + } + } else { + // Tuples and fixed-width vectors. + None + } + } else { + // Scalar types. + None + } + } +} + +impl FromStr for TypeKind { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(match s { + s if s.starts_with('{') && s.ends_with('}') => { + Self::Wildcard(s[1..s.len() - 1].trim().parse()?) + } + s if s.starts_with('*') => { + let mut split = s[1..].split_whitespace(); + let (ty, rw) = match (split.clone().count(), split.next(), split.next()) { + (2, Some("mut"), Some(ty)) => (ty, AccessLevel::RW), + (2, Some("const"), Some(ty)) => (ty, AccessLevel::R), + (1, Some(ty), None) => (ty, AccessLevel::R), + _ => return Err(format!("invalid pointer type {s:#?} given")), + }; + Self::Pointer(Box::new(ty.parse()?), rw) + } + _ => s + .parse::() + .map(TypeKind::Vector) + .or_else(|_| s.parse::().map(TypeKind::Base)) + .unwrap_or_else(|_| TypeKind::Custom(s.to_string())), + }) + } +} + +impl fmt::Display for TypeKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Vector(ty) => write!(f, "{ty}"), + Self::Pointer(ty, _) => write!(f, "{ty}"), + Self::Base(ty) => write!(f, "{ty}"), + Self::Wildcard(w) => write!(f, "{{{w}}}"), + Self::Custom(s) => write!(f, "{s}"), + } + } +} + +impl ToRepr for TypeKind { + fn repr(&self, repr: TypeRepr) -> String { + match self { + Self::Vector(ty) => ty.repr(repr), + Self::Pointer(ty, _) => ty.repr(repr), + Self::Base(ty) => ty.repr(repr), + Self::Wildcard(w) => format!("{w}"), + Self::Custom(s) => s.to_string(), + } + } +} + +impl ToTokens for TypeKind { + fn to_tokens(&self, tokens: &mut TokenStream) { + if let Self::Pointer(_, rw) = self { + tokens.append_all(match rw { + AccessLevel::RW => quote! { *mut }, + AccessLevel::R => quote! { *const }, + }) + } + + tokens.append_all( + self.to_string() + .parse::() + .expect("invalid syntax"), + ) + } +} + +impl PartialOrd for TypeKind { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl From<&TypeKind> for usize { + fn from(ty: &TypeKind) -> Self { + match ty { + TypeKind::Base(_) => 1, + TypeKind::Pointer(_, _) => 2, + TypeKind::Vector(_) => 3, + TypeKind::Custom(_) => 4, + TypeKind::Wildcard(_) => 5, + } + } +} + +impl Ord for TypeKind { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + use std::cmp::Ordering::*; + + let self_int: usize = self.into(); + let other_int: usize = other.into(); + + if self_int == other_int { + match (self, other) { + (TypeKind::Base(ty1), TypeKind::Base(ty2)) => ty1.cmp(ty2), + (TypeKind::Pointer(ty1, _), TypeKind::Pointer(ty2, _)) => ty1.cmp(ty2), + (TypeKind::Vector(vt1), TypeKind::Vector(vt2)) => vt1.cmp(vt2), + (TypeKind::Custom(s1), TypeKind::Custom(s2)) => s1.cmp(s2), + (TypeKind::Wildcard(..), TypeKind::Wildcard(..)) => Equal, + _ => unreachable!(), + } + } else { + self_int.cmp(&other_int) + } + } +} + +impl VectorType { + pub fn base_type(&self) -> &BaseType { + &self.base_type + } + + pub fn base_type_mut(&mut self) -> &mut BaseType { + &mut self.base_type + } + + fn sanitise_lanes( + mut base_type: BaseType, + lanes: Option, + ) -> Result<(BaseType, u32), String> { + let lanes = match (base_type, lanes) { + (BaseType::Sized(BaseTypeKind::Bool, lanes), None) => { + base_type = BaseType::Sized(BaseTypeKind::Bool, VECTOR_FULL_REGISTER_SIZE / lanes); + lanes + } + (BaseType::Unsized(BaseTypeKind::Bool), None) => { + base_type = BaseType::Sized(BaseTypeKind::Bool, 8); + 16 + } + (BaseType::Sized(_, size), None) => VECTOR_FULL_REGISTER_SIZE / size, + (BaseType::Sized(_, size), Some(lanes)) => match size * lanes { + VECTOR_FULL_REGISTER_SIZE | VECTOR_HALF_REGISTER_SIZE => lanes, + _ => return Err("invalid number of lanes".to_string()), + }, + _ => return Err("cannot infer number of lanes".to_string()), + }; + + Ok((base_type, lanes)) + } + + pub fn make_from_base( + base_ty: BaseType, + is_scalable: bool, + tuple_size: Option, + ) -> VectorType { + if is_scalable { + if let BaseType::Sized(BaseTypeKind::Bool, size) = base_ty { + return Self::make_predicate_from_bitsize(size); + } + } + + let (base_type, lanes) = Self::sanitise_lanes(base_ty, None).unwrap(); + + VectorType { + base_type, + lanes, + is_scalable, + tuple_size, + } + } + + pub fn make_predicate_from_bitsize(size: u32) -> VectorType { + VectorType { + base_type: BaseType::Sized(BaseTypeKind::Bool, size), + lanes: (VECTOR_FULL_REGISTER_SIZE / size), + is_scalable: true, + tuple_size: None, + } + } + + pub fn cast_base_type_as(&mut self, ty: BaseType) { + self.base_type = ty + } + + pub fn lanes(&self) -> u32 { + self.lanes + } + + pub fn tuple_size(&self) -> Option { + self.tuple_size + } +} + +impl FromStr for VectorType { + type Err = String; + + fn from_str(s: &str) -> Result { + lazy_static! { + static ref RE: Regex = Regex::new(r"^(?:(?:sv(?P(?:uint|int|bool|float)(?:\d+)?))|(?:(?P(?:uint|int|bool|poly|float)(?:\d+)?)x(?P(?:\d+)?)))(?:x(?P2|3|4))?_t$").unwrap(); + } + + if let Some(c) = RE.captures(s) { + let (base_type, lanes) = Self::sanitise_lanes( + c.name("sv_ty") + .or_else(|| c.name("ty")) + .map(<&str>::from) + .map(BaseType::from_str) + .unwrap()?, + c.name("lanes") + .map(<&str>::from) + .map(u32::from_str) + .transpose() + .unwrap(), + ) + .map_err(|e| format!("invalid {s:#?} vector type: {e}"))?; + + let tuple_size = c + .name("tuple_size") + .map(<&str>::from) + .map(VectorTupleSize::from_str) + .transpose() + .unwrap(); + + let v = Ok(VectorType { + base_type, + is_scalable: c.name("sv_ty").is_some(), + lanes, + tuple_size, + }); + return v; + } else { + Err(format!("invalid vector type {s:#?} given")) + } + } +} + +impl ToRepr for VectorType { + fn repr(&self, repr: TypeRepr) -> String { + let make_llvm_repr = |show_unsigned| { + format!( + "{}v{}{}", + if self.is_scalable { "nx" } else { "" }, + self.lanes * (self.tuple_size.map(usize::from).unwrap_or(1) as u32), + match self.base_type { + BaseType::Sized(BaseTypeKind::UInt, size) if show_unsigned => + format!("u{size}"), + _ => self.base_type.llvm_machine_repr(), + } + ) + }; + + if matches!(repr, TypeRepr::ACLENotation) { + self.base_type.acle_notation_repr() + } else if matches!(repr, TypeRepr::LLVMMachine) { + make_llvm_repr(false) + } else if self.is_scalable { + match (self.base_type, self.lanes, self.tuple_size) { + (BaseType::Sized(BaseTypeKind::Bool, _), 16, _) => "svbool_t".to_string(), + (BaseType::Sized(BaseTypeKind::Bool, _), lanes, _) => format!("svbool{lanes}_t"), + (BaseType::Sized(_, size), lanes, _) + if VECTOR_FULL_REGISTER_SIZE != (size * lanes) => + { + // Special internal type case + make_llvm_repr(true) + } + (ty, _, None) => format!("sv{}_t", ty.c_repr()), + (ty, _, Some(tuple_size)) => format!("sv{}x{tuple_size}_t", ty.c_repr()), + } + } else { + match self.tuple_size { + Some(tuple_size) => format!( + "{}x{}x{}_t", + self.base_type.c_repr(), + self.lanes, + tuple_size + ), + None => format!("{}x{}_t", self.base_type.c_repr(), self.lanes), + } + } + } +} + +impl fmt::Display for VectorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.c_repr()) + } +} + +impl From for usize { + fn from(t: VectorTupleSize) -> Self { + match t { + VectorTupleSize::Two => 2, + VectorTupleSize::Three => 3, + VectorTupleSize::Four => 4, + } + } +} + +impl FromStr for VectorTupleSize { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "2" => Ok(Self::Two), + "3" => Ok(Self::Three), + "4" => Ok(Self::Four), + _ => Err(format!("invalid vector tuple size `{s}` provided")), + } + } +} + +impl TryFrom for VectorTupleSize { + type Error = String; + + fn try_from(value: usize) -> Result { + match value { + 2 => Ok(Self::Two), + 3 => Ok(Self::Three), + 4 => Ok(Self::Four), + _ => Err(format!("invalid vector tuple size `{value}` provided")), + } + } +} + +impl fmt::Display for VectorTupleSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", usize::from(*self)) + } +} + +impl FromStr for BaseTypeKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "float" | "f" => Ok(Self::Float), + "int" | "i" => Ok(Self::Int), + "uint" | "u" => Ok(Self::UInt), + "poly" | "p" => Ok(Self::Poly), + "bool" | "b" => Ok(Self::Bool), + _ => Err(format!("no match for {s}")), + } + } +} + +impl ToRepr for BaseTypeKind { + fn repr(&self, repr: TypeRepr) -> String { + match (repr, self) { + (TypeRepr::C, Self::Float) => "float", + (TypeRepr::C, Self::Int) => "int", + (TypeRepr::C, Self::UInt) => "uint", + (TypeRepr::C, Self::Poly) => "poly", + (TypeRepr::Rust | TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::Float) => "f", + (TypeRepr::Rust, Self::Int) | (TypeRepr::LLVMMachine, Self::Int | Self::UInt) => "i", + (TypeRepr::Rust | TypeRepr::ACLENotation, Self::UInt) => "u", + (TypeRepr::Rust | TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::Poly) => "p", + (TypeRepr::ACLENotation, Self::Int) => "s", + (TypeRepr::ACLENotation, Self::Bool) => "b", + (_, Self::Bool) => "bool", + _ => { + unreachable!("no base type kind available for representation {repr:?}") + } + } + .to_string() + } +} + +impl fmt::Display for BaseTypeKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.c_repr()) + } +} + +impl BaseType { + pub fn get_size(&self) -> Result { + match self { + Self::Sized(_, size) => Ok(*size), + _ => Err(format!("unexpected invalid base type given {self:#?}")), + } + } + + pub fn kind(&self) -> &BaseTypeKind { + match self { + BaseType::Sized(kind, _) | BaseType::Unsized(kind) => kind, + } + } + + pub fn is_bool(&self) -> bool { + self.kind() == &BaseTypeKind::Bool + } + + pub fn is_float(&self) -> bool { + self.kind() == &BaseTypeKind::Float + } +} + +impl FromStr for BaseType { + type Err = String; + + fn from_str(s: &str) -> Result { + lazy_static! { + static ref RE: Regex = Regex::new(r"^(?P[a-zA-Z]+)(?P\d+)?(_t)?$").unwrap(); + } + + if let Some(c) = RE.captures(s) { + let kind = c["kind"].parse()?; + let size = c + .name("size") + .map(<&str>::from) + .map(u32::from_str) + .transpose() + .unwrap(); + match size { + Some(size) => Ok(Self::Sized(kind, size)), + None => Ok(Self::Unsized(kind)), + } + } else { + Err(format!("failed to parse type `{s}`")) + } + } +} + +impl ToRepr for BaseType { + fn repr(&self, repr: TypeRepr) -> String { + use BaseType::*; + use BaseTypeKind::*; + use TypeRepr::*; + match (self, &repr) { + (Sized(Bool, _) | Unsized(Bool), LLVMMachine) => "i1".to_string(), + (Sized(_, size), SizeLiteral) if *size == 8 => "b".to_string(), + (Sized(_, size), SizeLiteral) if *size == 16 => "h".to_string(), + (Sized(_, size), SizeLiteral) if *size == 32 => "w".to_string(), + (Sized(_, size), SizeLiteral) if *size == 64 => "d".to_string(), + (Sized(_, size), SizeLiteral) if *size == 128 => "q".to_string(), + (_, SizeLiteral) => unreachable!("cannot represent {self:#?} as size literal"), + (Sized(Float, _) | Unsized(Float), TypeKind) => "f".to_string(), + (Sized(Int, _) | Unsized(Int), TypeKind) => "s".to_string(), + (Sized(UInt, _) | Unsized(UInt), TypeKind) => "u".to_string(), + (Sized(_, size), Size) => size.to_string(), + (Sized(_, size), SizeInBytesLog2) => { + assert!(size.is_power_of_two() && *size >= 8); + (size >> 3).trailing_zeros().to_string() + } + (Sized(kind, size), _) => format!("{}{size}", kind.repr(repr)), + (Unsized(kind), _) => kind.repr(repr), + } + } +} + +impl fmt::Display for BaseType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.rust_repr()) + } +} + +#[cfg(test)] +mod tests { + use crate::typekinds::*; + + #[test] + fn test_predicate() { + assert_eq!( + "svbool_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::Bool, 8), + is_scalable: true, + lanes: 16, + tuple_size: None + }) + ); + } + + #[test] + fn test_llvm_internal_predicate() { + assert_eq!( + "svbool4_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::Bool, 32), + is_scalable: true, + lanes: 4, + tuple_size: None + }) + ); + } + + #[test] + fn test_llvm_internal_predicate_llvm() { + assert_eq!( + "svbool4_t".parse::().unwrap().llvm_machine_repr(), + "nxv4i1" + ); + } + + #[test] + fn test_llvm_internal_predicate_acle() { + assert_eq!( + "svbool4_t" + .parse::() + .unwrap() + .acle_notation_repr(), + "b32" + ); + } + + #[test] + fn test_predicate_from_bitsize() { + let pg = VectorType::make_predicate_from_bitsize(32); + assert_eq!(pg.acle_notation_repr(), "b32"); + assert_eq!(pg, "svbool4_t".parse().unwrap()); + assert_eq!(pg.lanes, 4); + assert_eq!(pg.base_type, BaseType::Sized(BaseTypeKind::Bool, 32)); + } + + #[test] + fn test_scalable_single() { + assert_eq!( + "svuint8_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 8), + is_scalable: true, + lanes: 16, + tuple_size: None + }) + ); + } + + #[test] + fn test_scalable_tuple() { + assert_eq!( + "svint64x3_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::Int, 64), + is_scalable: true, + lanes: 2, + tuple_size: Some(VectorTupleSize::Three), + }) + ); + } + + #[test] + fn test_scalable_single_llvm() { + assert_eq!( + "svuint32_t" + .parse::() + .unwrap() + .llvm_machine_repr(), + "nxv4i32" + ); + } + + #[test] + fn test_scalable_tuple_llvm() { + assert_eq!( + "svint32x4_t" + .parse::() + .unwrap() + .llvm_machine_repr(), + "nxv16i32" + ); + } + + #[test] + fn test_vector_single_full() { + assert_eq!( + "uint32x4_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 32), + is_scalable: false, + lanes: 4, + tuple_size: None, + }) + ); + } + + #[test] + fn test_vector_single_half() { + assert_eq!( + "uint32x2_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 32), + is_scalable: false, + lanes: 2, + tuple_size: None, + }) + ); + } + + #[test] + fn test_vector_tuple() { + assert_eq!( + "uint64x2x4_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 64), + is_scalable: false, + lanes: 2, + tuple_size: Some(VectorTupleSize::Four), + }) + ); + } + + #[test] + fn test_const_pointer() { + let p = "*u32".parse::().unwrap(); + assert_eq!( + p, + TypeKind::Pointer( + Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32))), + AccessLevel::R + ) + ); + assert_eq!(p.to_token_stream().to_string(), "* const u32") + } + + #[test] + fn test_mut_pointer() { + let p = "*mut u32".parse::().unwrap(); + assert_eq!( + p, + TypeKind::Pointer( + Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32))), + AccessLevel::RW + ) + ); + assert_eq!(p.to_token_stream().to_string(), "* mut u32") + } + + #[test] + #[should_panic] + fn test_invalid_vector_single() { + assert_eq!( + "uint32x8_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 32), + is_scalable: false, + lanes: 8, + tuple_size: None, + }) + ); + } + + #[test] + #[should_panic] + fn test_invalid_vector_tuple() { + assert_eq!( + "uint32x4x5_t".parse::().unwrap(), + TypeKind::Vector(VectorType { + base_type: BaseType::Sized(BaseTypeKind::UInt, 32), + is_scalable: false, + lanes: 8, + tuple_size: None, // cannot represent + }) + ); + } + + #[test] + fn test_base() { + assert_eq!( + "u32".parse::().unwrap(), + TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32)), + ) + } + + #[test] + fn test_custom() { + assert_eq!( + "svpattern".parse::().unwrap(), + TypeKind::Custom("svpattern".to_string()), + ) + } + + #[test] + fn test_wildcard_type() { + assert_eq!( + "{type}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::Type(None)), + ) + } + + #[test] + fn test_wildcard_typeset() { + assert_eq!( + "{type[0]}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::Type(Some(0))), + ) + } + + #[test] + fn test_wildcard_sve_type() { + assert_eq!( + "{sve_type}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::SVEType(None, None)), + ) + } + + #[test] + fn test_wildcard_sve_typeset() { + assert_eq!( + "{sve_type[0]}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::SVEType(Some(0), None)), + ) + } + + #[test] + fn test_wildcard_sve_tuple_type() { + assert_eq!( + "{sve_type_x2}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::SVEType(None, Some(VectorTupleSize::Two))), + ) + } + + #[test] + fn test_wildcard_sve_tuple_typeset() { + assert_eq!( + "{sve_type_x2[0]}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::SVEType(Some(0), Some(VectorTupleSize::Two))), + ) + } + + #[test] + fn test_wildcard_predicate() { + assert_eq!( + "{predicate}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::Predicate(None)) + ) + } + + #[test] + fn test_wildcard_scale() { + assert_eq!( + "{sve_type as i8}".parse::().unwrap(), + TypeKind::Wildcard(Wildcard::Scale( + Box::new(Wildcard::SVEType(None, None)), + Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::Int, 8))) + )) + ) + } + + #[test] + fn test_size_in_bytes_log2() { + assert_eq!("i8".parse::().unwrap().size_in_bytes_log2(), "0"); + assert_eq!("i16".parse::().unwrap().size_in_bytes_log2(), "1"); + assert_eq!("i32".parse::().unwrap().size_in_bytes_log2(), "2"); + assert_eq!("i64".parse::().unwrap().size_in_bytes_log2(), "3") + } + + #[test] + #[should_panic] + fn test_invalid_size_in_bytes_log2() { + "i9".parse::().unwrap().size_in_bytes_log2(); + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/wildcards.rs b/library/stdarch/crates/stdarch-gen-arm/src/wildcards.rs new file mode 100644 index 00000000000..25aa8034892 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/wildcards.rs @@ -0,0 +1,197 @@ +use lazy_static::lazy_static; +use regex::Regex; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use std::fmt; +use std::str::FromStr; + +use crate::{ + fn_suffix::SuffixKind, + predicate_forms::PredicationMask, + typekinds::{ToRepr, TypeKind, TypeKindOptions, VectorTupleSize}, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] +pub enum Wildcard { + Type(Option), + /// NEON type derivated by a base type + NEONType(Option, Option, Option), + /// SVE type derivated by a base type + SVEType(Option, Option), + /// Integer representation of bitsize + Size(Option), + /// Integer representation of bitsize minus one + SizeMinusOne(Option), + /// Literal representation of the bitsize: b(yte), h(half), w(ord) or d(ouble) + SizeLiteral(Option), + /// Literal representation of the type kind: f(loat), s(igned), u(nsigned) + TypeKind(Option, Option), + /// Log2 of the size in bytes + SizeInBytesLog2(Option), + /// Predicate to be inferred from the specified type + Predicate(Option), + /// Predicate to be inferred from the greatest type + MaxPredicate, + + Scale(Box, Box), + + // Other wildcards + LLVMLink, + NVariant, + /// Predicate forms to use and placeholder for a predicate form function name modifier + PredicateForms(PredicationMask), + + /// User-set wildcard through `substitutions` + Custom(String), +} + +impl Wildcard { + pub fn is_nonpredicate_type(&self) -> bool { + matches!( + self, + Wildcard::Type(..) | Wildcard::NEONType(..) | Wildcard::SVEType(..) + ) + } + + pub fn get_typeset_index(&self) -> Option { + match self { + Wildcard::Type(idx) | Wildcard::NEONType(idx, ..) | Wildcard::SVEType(idx, ..) => { + Some(idx.unwrap_or(0)) + } + _ => None, + } + } +} + +impl FromStr for Wildcard { + type Err = String; + + fn from_str(s: &str) -> Result { + lazy_static! { + static ref RE: Regex = Regex::new(r"^(?P\w+?)(?:_x(?P[2-4]))?(?:\[(?P\d+)\])?(?:\.(?P\w+))?(?:\s+as\s+(?P.*?))?$").unwrap(); + } + + if let Some(c) = RE.captures(s) { + let wildcard_name = &c["wildcard"]; + let inputset_index = c + .name("index") + .map(<&str>::from) + .map(usize::from_str) + .transpose() + .map_err(|_| format!("{:#?} is not a valid type index", &c["index"]))?; + let tuple_size = c + .name("tuple_size") + .map(<&str>::from) + .map(VectorTupleSize::from_str) + .transpose() + .map_err(|_| format!("{:#?} is not a valid tuple size", &c["tuple_size"]))?; + let modifiers = c.name("modifiers").map(<&str>::from); + + let wildcard = match (wildcard_name, inputset_index, tuple_size, modifiers) { + ("type", index, None, None) => Ok(Wildcard::Type(index)), + ("neon_type", index, tuple, modifier) => { + if let Some(str_suffix) = modifier { + let suffix_kind = SuffixKind::from_str(str_suffix); + return Ok(Wildcard::NEONType(index, tuple, Some(suffix_kind.unwrap()))); + } else { + Ok(Wildcard::NEONType(index, tuple, None)) + } + } + ("sve_type", index, tuple, None) => Ok(Wildcard::SVEType(index, tuple)), + ("size", index, None, None) => Ok(Wildcard::Size(index)), + ("size_minus_one", index, None, None) => Ok(Wildcard::SizeMinusOne(index)), + ("size_literal", index, None, None) => Ok(Wildcard::SizeLiteral(index)), + ("type_kind", index, None, modifiers) => Ok(Wildcard::TypeKind( + index, + modifiers.map(|modifiers| modifiers.parse()).transpose()?, + )), + ("size_in_bytes_log2", index, None, None) => Ok(Wildcard::SizeInBytesLog2(index)), + ("predicate", index, None, None) => Ok(Wildcard::Predicate(index)), + ("max_predicate", None, None, None) => Ok(Wildcard::MaxPredicate), + ("llvm_link", None, None, None) => Ok(Wildcard::LLVMLink), + ("_n", None, None, None) => Ok(Wildcard::NVariant), + (w, None, None, None) if w.starts_with('_') => { + // test for predicate forms + let pf_mask = PredicationMask::from_str(&w[1..]); + if let Ok(mask) = pf_mask { + if mask.has_merging() { + Ok(Wildcard::PredicateForms(mask)) + } else { + Err("cannot add predication without a Merging form".to_string()) + } + } else { + Err(format!("invalid wildcard `{s:#?}`")) + } + } + (cw, None, None, None) => Ok(Wildcard::Custom(cw.to_string())), + _ => Err(format!("invalid wildcard `{s:#?}`")), + }?; + + let scale_to = c + .name("scale_to") + .map(<&str>::from) + .map(TypeKind::from_str) + .transpose() + .map_err(|_| format!("{:#?} is not a valid type", &c["scale_to"]))?; + + if let Some(scale_to) = scale_to { + Ok(Wildcard::Scale(Box::new(wildcard), Box::new(scale_to))) + } else { + Ok(wildcard) + } + } else { + Err(format!("## invalid wildcard `{s:#?}`")) + } + } +} + +impl fmt::Display for Wildcard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Type(None) => write!(f, "type"), + Self::Type(Some(index)) => write!(f, "type[{index}]"), + Self::NEONType(None, None, None) => write!(f, "neon_type"), + Self::NEONType(None, None, Some(suffix_kind)) => write!(f, "neon_type.{suffix_kind}"), + Self::NEONType(Some(index), None, None) => write!(f, "neon_type[{index}]"), + Self::NEONType(Some(index), None, Some(suffix_kind)) => { + write!(f, "neon_type[{index}].{suffix_kind}") + } + Self::NEONType(None, Some(tuple_size), Some(suffix_kind)) => { + write!(f, "neon_type_x{tuple_size}.{suffix_kind}") + } + Self::NEONType(None, Some(tuple_size), None) => write!(f, "neon_type_x{tuple_size}"), + Self::NEONType(Some(index), Some(tuple_size), None) => { + write!(f, "neon_type_x{tuple_size}[{index}]") + } + Self::NEONType(Some(index), Some(tuple_size), Some(suffix_kind)) => { + write!(f, "neon_type_x{tuple_size}[{index}].{suffix_kind}") + } + Self::SVEType(None, None) => write!(f, "sve_type"), + Self::SVEType(Some(index), None) => write!(f, "sve_type[{index}]"), + Self::SVEType(None, Some(tuple_size)) => write!(f, "sve_type_x{tuple_size}"), + Self::SVEType(Some(index), Some(tuple_size)) => { + write!(f, "sve_type_x{tuple_size}[{index}]") + } + Self::Size(None) => write!(f, "size"), + Self::Size(Some(index)) => write!(f, "size[{index}]"), + Self::SizeMinusOne(None) => write!(f, "size_minus_one"), + Self::SizeMinusOne(Some(index)) => write!(f, "size_minus_one[{index}]"), + Self::SizeLiteral(None) => write!(f, "size_literal"), + Self::SizeLiteral(Some(index)) => write!(f, "size_literal[{index}]"), + Self::TypeKind(None, None) => write!(f, "type_kind"), + Self::TypeKind(None, Some(opts)) => write!(f, "type_kind.{opts}"), + Self::TypeKind(Some(index), None) => write!(f, "type_kind[{index}]"), + Self::TypeKind(Some(index), Some(opts)) => write!(f, "type_kind[{index}].{opts}"), + Self::SizeInBytesLog2(None) => write!(f, "size_in_bytes_log2"), + Self::SizeInBytesLog2(Some(index)) => write!(f, "size_in_bytes_log2[{index}]"), + Self::Predicate(None) => write!(f, "predicate"), + Self::Predicate(Some(index)) => write!(f, "predicate[{index}]"), + Self::MaxPredicate => write!(f, "max_predicate"), + Self::LLVMLink => write!(f, "llvm_link"), + Self::NVariant => write!(f, "_n"), + Self::PredicateForms(mask) => write!(f, "_{mask}"), + + Self::Scale(wildcard, ty) => write!(f, "{wildcard} as {}", ty.rust_repr()), + Self::Custom(cw) => write!(f, "{cw}"), + } + } +} diff --git a/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs new file mode 100644 index 00000000000..2eb467b9628 --- /dev/null +++ b/library/stdarch/crates/stdarch-gen-arm/src/wildstring.rs @@ -0,0 +1,399 @@ +use itertools::Itertools; +use proc_macro2::TokenStream; +use quote::{quote, ToTokens, TokenStreamExt}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; +use std::str::pattern::Pattern; +use std::{fmt, str::FromStr}; + +use crate::context::LocalContext; +use crate::fn_suffix::make_neon_suffix; +use crate::typekinds::{ToRepr, TypeRepr}; +use crate::wildcards::Wildcard; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WildStringPart { + String(String), + Wildcard(Wildcard), +} + +/// Wildcard-able string +#[derive(Debug, Clone, PartialEq, Eq, Default, SerializeDisplay, DeserializeFromStr)] +pub struct WildString(pub Vec); + +impl WildString { + pub fn has_wildcards(&self) -> bool { + for part in self.0.iter() { + if let WildStringPart::Wildcard(..) = part { + return true; + } + } + + false + } + + pub fn wildcards(&self) -> impl Iterator + '_ { + self.0.iter().filter_map(|part| match part { + WildStringPart::Wildcard(w) => Some(w), + _ => None, + }) + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator + '_ { + self.0.iter_mut() + } + + pub fn starts_with(&self, s2: &str) -> bool { + self.to_string().starts_with(s2) + } + + pub fn prepend_str(&mut self, s: impl Into) { + self.0.insert(0, WildStringPart::String(s.into())) + } + + pub fn push_str(&mut self, s: impl Into) { + self.0.push(WildStringPart::String(s.into())) + } + + pub fn push_wildcard(&mut self, w: Wildcard) { + self.0.push(WildStringPart::Wildcard(w)) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn replace<'a, P>(&'a self, from: P, to: &str) -> WildString + where + P: Pattern + Copy, + { + WildString( + self.0 + .iter() + .map(|part| match part { + WildStringPart::String(s) => WildStringPart::String(s.replace(from, to)), + part => part.clone(), + }) + .collect_vec(), + ) + } + + pub fn build_acle(&mut self, ctx: &LocalContext) -> Result<(), String> { + self.build(ctx, TypeRepr::ACLENotation) + } + + pub fn build_neon_intrinsic_signature(&mut self, ctx: &LocalContext) -> Result<(), String> { + let repr = TypeRepr::ACLENotation; + self.iter_mut().try_for_each(|wp| -> Result<(), String> { + if let WildStringPart::Wildcard(w) = wp { + match w { + Wildcard::NEONType(_, _, ref maybe_suffix_kind) => { + if let Some(suffix_kind) = maybe_suffix_kind { + let x = ctx.provide_type_wildcard(w).unwrap(); + *wp = WildStringPart::String(make_neon_suffix(x, *suffix_kind)) + } else { + *wp = WildString::make_default_build(ctx, repr, w) + } + } + _ => *wp = WildString::make_default_build(ctx, repr, w), + } + } + Ok(()) + }) + } + + pub fn build(&mut self, ctx: &LocalContext, repr: TypeRepr) -> Result<(), String> { + match repr { + TypeRepr::ACLENotation | TypeRepr::LLVMMachine => { + self.iter_mut().try_for_each(|wp| -> Result<(), String> { + if let WildStringPart::Wildcard(w) = wp { + match w { + Wildcard::NEONType(_, _, ref maybe_suffix_kind) => { + if let Some(suffix_kind) = maybe_suffix_kind { + let x = ctx.provide_type_wildcard(w).unwrap(); + *wp = WildStringPart::String(make_neon_suffix(x, *suffix_kind)) + } else { + *wp = WildString::make_default_build(ctx, repr, w) + } + } + _ => *wp = WildString::make_default_build(ctx, repr, w), + } + } + Ok(()) + }) + } + _ => self.iter_mut().try_for_each(|wp| -> Result<(), String> { + if let WildStringPart::Wildcard(w) = wp { + *wp = WildString::make_default_build(ctx, repr, w); + } + Ok(()) + }), + } + } + + fn make_default_build(ctx: &LocalContext, repr: TypeRepr, w: &mut Wildcard) -> WildStringPart { + WildStringPart::String( + ctx.provide_substitution_wildcard(w) + .or_else(|_| ctx.provide_type_wildcard(w).map(|ty| ty.repr(repr))) + .unwrap(), + ) + } +} + +impl From for WildString { + fn from(s: String) -> Self { + WildString(vec![WildStringPart::String(s)]) + } +} + +impl FromStr for WildString { + type Err = String; + + fn from_str(s: &str) -> Result { + enum State { + Normal { start: usize }, + Wildcard { start: usize, count: usize }, + EscapeTokenOpen { start: usize, at: usize }, + EscapeTokenClose { start: usize, at: usize }, + } + + let mut ws = WildString::default(); + match s + .char_indices() + .try_fold(State::Normal { start: 0 }, |state, (idx, ch)| { + match (state, ch) { + (State::Normal { start }, '{') => Ok(State::EscapeTokenOpen { start, at: idx }), + (State::Normal { start }, '}') => { + Ok(State::EscapeTokenClose { start, at: idx }) + } + (State::EscapeTokenOpen { start, at }, '{') + | (State::EscapeTokenClose { start, at }, '}') => { + if start < at { + ws.push_str(&s[start..at]) + } + + Ok(State::Normal { start: idx }) + } + (State::EscapeTokenOpen { at, .. }, '}') => Err(format!( + "empty wildcard given in string {s:?} at position {at}" + )), + (State::EscapeTokenOpen { start, at }, _) => { + if start < at { + ws.push_str(&s[start..at]) + } + + Ok(State::Wildcard { + start: idx, + count: 0, + }) + } + (State::EscapeTokenClose { at, .. }, _) => Err(format!( + "closing a non-wildcard/bad escape in string {s:?} at position {at}" + )), + // Nesting wildcards is only supported for `{foo as {bar}}`, wildcards cannot be + // nested at the start of a WildString. + (State::Wildcard { start, count }, '{') => Ok(State::Wildcard { + start, + count: count + 1, + }), + (State::Wildcard { start, count: 0 }, '}') => { + ws.push_wildcard(s[start..idx].parse()?); + Ok(State::Normal { start: idx + 1 }) + } + (State::Wildcard { start, count }, '}') => Ok(State::Wildcard { + start, + count: count - 1, + }), + (state @ State::Normal { .. }, _) | (state @ State::Wildcard { .. }, _) => { + Ok(state) + } + } + })? { + State::Normal { start } => { + if start < s.len() { + ws.push_str(&s[start..]); + } + + Ok(ws) + } + State::EscapeTokenOpen { at, .. } | State::Wildcard { start: at, .. } => Err(format!( + "unclosed wildcard in string {s:?} at position {at}" + )), + State::EscapeTokenClose { at, .. } => Err(format!( + "closing a non-wildcard/bad escape in string {s:?} at position {at}" + )), + } + } +} + +impl fmt::Display for WildString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + self.0 + .iter() + .map(|part| match part { + WildStringPart::String(s) => s.to_owned(), + WildStringPart::Wildcard(w) => format!("{{{w}}}"), + }) + .join("") + ) + } +} + +impl ToTokens for WildString { + fn to_tokens(&self, tokens: &mut TokenStream) { + assert!( + !self.has_wildcards(), + "cannot convert string with wildcards {self:?} to TokenStream" + ); + let str = self.to_string(); + tokens.append_all(quote! { #str }) + } +} + +#[cfg(test)] +mod tests { + use crate::typekinds::*; + use crate::wildstring::*; + + #[test] + fn test_empty_string() { + let ws: WildString = "".parse().unwrap(); + assert_eq!(ws.0.len(), 0); + } + + #[test] + fn test_plain_string() { + let ws: WildString = "plain string".parse().unwrap(); + assert_eq!(ws.0.len(), 1); + assert_eq!( + ws, + WildString(vec![WildStringPart::String("plain string".to_string())]) + ) + } + + #[test] + fn test_escaped_curly_brackets() { + let ws: WildString = "VALUE = {{value}}".parse().unwrap(); + assert_eq!(ws.to_string(), "VALUE = {value}"); + assert!(!ws.has_wildcards()); + } + + #[test] + fn test_escaped_curly_brackets_wildcard() { + let ws: WildString = "TYPE = {{{type}}}".parse().unwrap(); + assert_eq!(ws.to_string(), "TYPE = {{type}}"); + assert_eq!(ws.0.len(), 4); + assert!(ws.has_wildcards()); + } + + #[test] + fn test_wildcard_right_boundary() { + let s = "string test {type}"; + let ws: WildString = s.parse().unwrap(); + assert_eq!(&ws.to_string(), s); + assert!(ws.has_wildcards()); + } + + #[test] + fn test_wildcard_left_boundary() { + let s = "{type} string test"; + let ws: WildString = s.parse().unwrap(); + assert_eq!(&ws.to_string(), s); + assert!(ws.has_wildcards()); + } + + #[test] + fn test_recursive_wildcard() { + let s = "string test {type[0] as {type[1]}}"; + let ws: WildString = s.parse().unwrap(); + + assert_eq!(ws.0.len(), 2); + assert_eq!( + ws, + WildString(vec![ + WildStringPart::String("string test ".to_string()), + WildStringPart::Wildcard(Wildcard::Scale( + Box::new(Wildcard::Type(Some(0))), + Box::new(TypeKind::Wildcard(Wildcard::Type(Some(1)))), + )) + ]) + ); + } + + #[test] + fn test_scale_wildcard() { + let s = "string {type[0] as i8} test"; + let ws: WildString = s.parse().unwrap(); + + assert_eq!(ws.0.len(), 3); + assert_eq!( + ws, + WildString(vec![ + WildStringPart::String("string ".to_string()), + WildStringPart::Wildcard(Wildcard::Scale( + Box::new(Wildcard::Type(Some(0))), + Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::Int, 8))), + )), + WildStringPart::String(" test".to_string()) + ]) + ); + } + + #[test] + fn test_solitaire_wildcard() { + let ws: WildString = "{type}".parse().unwrap(); + assert_eq!(ws.0.len(), 1); + assert_eq!( + ws, + WildString(vec![WildStringPart::Wildcard(Wildcard::Type(None))]) + ) + } + + #[test] + fn test_empty_wildcard() { + "string {}" + .parse::() + .expect_err("expected parse error"); + } + + #[test] + fn test_invalid_open_wildcard_right() { + "string {" + .parse::() + .expect_err("expected parse error"); + } + + #[test] + fn test_invalid_close_wildcard_right() { + "string }" + .parse::() + .expect_err("expected parse error"); + } + + #[test] + fn test_invalid_open_wildcard_left() { + "{string" + .parse::() + .expect_err("expected parse error"); + } + + #[test] + fn test_invalid_close_wildcard_left() { + "}string" + .parse::() + .expect_err("expected parse error"); + } + + #[test] + fn test_consecutive_wildcards() { + let s = "svprf{size_literal[1]}_gather_{type[0]}{index_or_offset}"; + let ws: WildString = s.parse().unwrap(); + assert_eq!(ws.to_string(), s) + } +} diff --git a/library/stdarch/crates/stdarch-gen2/Cargo.toml b/library/stdarch/crates/stdarch-gen2/Cargo.toml deleted file mode 100644 index c9a039ea6b7..00000000000 --- a/library/stdarch/crates/stdarch-gen2/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "stdarch-gen2" -version = "0.1.0" -authors = ["Luca Vizzarro ", - "Jamie Cunliffe ", - "Adam Gemmell "] -license = "MIT OR Apache-2.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -itertools = "0.10" -lazy_static = "1.4.0" -proc-macro2 = "1.0" -quote = "1.0" -regex = "1.5" -serde = { version = "1.0", features = ["derive"] } -serde_with = "1.14" -serde_yaml = "0.8" -walkdir = "2.3.2" diff --git a/library/stdarch/crates/stdarch-gen2/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen2/spec/neon/aarch64.spec.yml deleted file mode 100644 index fd704c305e3..00000000000 --- a/library/stdarch/crates/stdarch-gen2/spec/neon/aarch64.spec.yml +++ /dev/null @@ -1,9852 +0,0 @@ -arch_cfgs: - - arch_name: aarch64 - target_feature: [neon] - llvm_prefix: llvm.aarch64.neon - -# Repeatedly used anchors -# #[stable(feature = "neon_intrinsics", since = "1.59.0")] -neon-stable: &neon-stable - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - -# #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] -neon-unstable: &neon-unstable - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - -# #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -neon-v7: &neon-v7 - FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v7"']]} ]] - -# #[target_feature(enable = "neon,v7")] -enable-v7: &enable-v7 - FnCall: [target_feature, ['enable = "neon,v7"']] - -# #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] -neon-v8: &neon-v8 - FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v8"']]} ]] - -target-is-arm: &target-is-arm - FnCall: [cfg, ['target_arch = "arm"']] - -# #[cfg(not(target_arch = "arm"))] -target-not-arm: &target-not-arm - FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm"']]}]] - -neon-target-aarch64-arm64ec: &neon-target-aarch64-arm64ec - FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]] - -# #[cfg_attr(not(target_arch = "arm"), stable(feature = "neon_intrinsics", since = "1.59.0"))] -neon-stable-not-arm: &neon-stable-not-arm - FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']]}, *neon-stable]] - -#[cfg_attr(target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800"))] -neon-unstable-is-arm: &neon-unstable-is-arm - FnCall: [ cfg_attr, ['target_arch = "arm"', *neon-unstable]] - -# #[cfg_attr(all(test, not(target_env = "msvc"))] -msvc-disabled: &msvc-disabled - FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]] - -# all(test, target_arch = "arm") -test-is-arm: &test-is-arm - FnCall: [all, [test, 'target_arch = "arm"']] - -# #[target_feature(enable = "neon,aes")] -neon-aes: &neon-aes - FnCall: [target_feature, ['enable = "neon,aes"']] - -# #[target_feature(enable = "neon,i8mm")] -neon-i8mm: &neon-i8mm - FnCall: [target_feature, ['enable = "neon,i8mm"']] - -#[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_i8mm", issue = "117223"))] -neon-unstable-i8mm: &neon-unstable-i8mm - FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']] }, { FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] } ]] - -# #[unstable(feature = "stdarch_neon_fcma", issue = "117222")] -neon-unstable-fcma: &neon-unstable-fcma - FnCall: [unstable, ['feature = "stdarch_neon_fcma"', 'issue = "117222"']] - -intrinsics: - - name: "vaddd_{type}" - doc: Add - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - i64 - - u64 - compose: - - MethodCall: - - a - - wrapping_add - - - b - - - name: "veor3{neon_type.no}" - doc: Three-way exclusive OR - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - assert_instr: [eor3] - safety: - unsafe: [neon] - types: - - int8x16_t - - int16x8_t - - int32x4_t - - int64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.crypto.eor3s.{neon_type}" - links: - - link: "llvm.aarch64.crypto.eor3s.{neon_type}" - arch: aarch64,arm64ec - - - name: "veor3{neon_type.no}" - doc: Three-way exclusive OR - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - assert_instr: [eor3] - safety: - unsafe: [neon] - types: - - uint8x16_t - - uint16x8_t - - uint32x4_t - - uint64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.crypto.eor3u.{neon_type}" - links: - - link: "llvm.aarch64.crypto.eor3u.{neon_type}" - arch: aarch64,arm64ec - - - name: "vabd{neon_type.no}" - doc: Absolute difference between the arguments of Floating - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fabd] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fabd.{neon_type}" - links: - - link: "llvm.aarch64.neon.fabd.{neon_type}" - arch: aarch64,arm64ec - - - name: "vabd{type[0]}" - doc: "Floating-point absolute difference" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [fabd] - safety: - unsafe: [neon] - types: - - ['s_f32', 'f32'] - - ['d_f64', 'f64'] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vabd_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[1]}", [b]] - - 0 - - - name: "vabdl_high{neon_type[0].noq}" - doc: Signed Absolute difference Long - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [sabdl] - safety: - unsafe: [neon] - types: - - [int8x16_t, int16x8_t, int8x8_t, uint8x8_t] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - a - - a - - [8, 9, 10, 11, 12, 13, 14, 15] - - Let: - - d - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - b - - b - - [8, 9, 10, 11, 12, 13, 14, 15] - - Let: - - e - - "{neon_type[3]}" - - FnCall: - - simd_cast - - - FnCall: - - "vabd_{neon_type[0]}" - - - c - - d - - FnCall: - - simd_cast - - - e - - - name: "vabdl_high{neon_type[0].noq}" - doc: Signed Absolute difference Long - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - assert_instr: [sabdl] - safety: - unsafe: [neon] - types: - - [int16x8_t, int32x4_t, int16x4_t, uint16x4_t] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - a - - a - - [4, 5, 6, 7] - - Let: - - d - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - b - - b - - [4, 5, 6, 7] - - Let: - - e - - "{neon_type[3]}" - - FnCall: - - simd_cast - - - FnCall: - - "vabd_{neon_type[0]}" - - - c - - d - - FnCall: - - simd_cast - - - e - - - name: "vabdl_high{neon_type[0].noq}" - doc: Signed Absolute difference Long - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - assert_instr: [sabdl] - safety: - unsafe: [neon] - types: - - [int32x4_t, int64x2_t, int32x2_t, uint32x2_t] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - a - - a - - [2, 3] - - Let: - - d - - "{neon_type[2]}" - - FnCall: - - simd_shuffle! - - - b - - b - - [2, 3] - - Let: - - e - - "{neon_type[3]}" - - FnCall: - - simd_cast - - - FnCall: - - "vabd_{neon_type[0]}" - - - c - - d - - FnCall: - - simd_cast - - - e - - - name: "vceq{neon_type[0].no}" - doc: "Compare bitwise Equal (vector)" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint64x1_t, uint64x1_t] - - [uint64x2_t, uint64x2_t] - - [int64x1_t, uint64x1_t] - - [int64x2_t, uint64x2_t] - - [poly64x1_t, uint64x1_t] - - [poly64x2_t, uint64x2_t] - compose: - - FnCall: [simd_eq, [a, b]] - - - name: "vceq{neon_type[0].no}" - doc: "Floating-point compare equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmeq]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: [simd_eq, [a, b]] - - - name: "vceq{type[0]}" - doc: "Floating-point compare equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vceq_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[1]}", [b]] - - '0' - - - name: "vceqd_{type[0]}" - doc: "Compare bitwise equal" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64", "s64"] - - ["u64", "u64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - "vceq_{type[2]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vtst{neon_type[0].no}" - doc: "Signed compare bitwise Test bits nonzero" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmtst]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)'] - - [poly64x1_t, uint64x1_t, 'i64x1', 'i64x1::new(0)'] - - [poly64x2_t, uint64x2_t, 'i64x2', 'i64x2::new(0, 0)'] - compose: - - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] - - Let: [d, "{type[2]}", "{type[3]}"] - - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] - - - name: "vtstd_{type[0]}" - doc: "Compare bitwise test bits nonzero" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [tst]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64", "s64"] - - ["u64", "u64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - "vtst_{type[2]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vuqadd{type[0]}" - doc: "Signed saturating accumulate of unsigned value" - arguments: ["a: {type[1]}", "b: {type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [suqadd]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_s32", "i32", "u32"] - - ["d_s64", "i64", "u64"] - compose: - - LLVMLink: - name: "vuqadd{type[0]}" - links: - - link: "llvm.aarch64.neon.suqadd.{type[1]}" - arch: aarch64,arm64ec - - - name: "vuqadd{type[0]}" - doc: "Signed saturating accumulate of unsigned value" - arguments: ["a: {type[1]}", "b: {type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [suqadd]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["b_s8", "i8", "u8", "s8"] - - ["h_s16", "i16", "u16", "s16"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vuqadd_{type[3]}" - - - FnCall: ["vdup_n_{type[3]}", [a]] - - FnCall: ["vdup_n_{type[2]}", [b]] - - '0' - - - name: "vabs{neon_type.no}" - doc: "Floating-point absolute value" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fabs]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_fabs, [a]] - - - name: "vcgt{neon_type[0].no}" - doc: "Compare signed greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, uint64x1_t] - - [int64x2_t, uint64x2_t] - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vcgt{neon_type.no}" - doc: "Compare unsigned greater than" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhi]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - uint64x1_t - - uint64x2_t - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vcgt{neon_type[0].no}" - doc: "Floating-point compare greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vcgt{type[0]}" - doc: "Floating-point compare greater than" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - 'simd_extract!' - - - FnCall: - - "vcgt_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[1]}", [b]] - - '0' - - - name: "vclt{neon_type[0].no}" - doc: "Compare signed less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, uint64x1_t] - - [int64x2_t, uint64x2_t] - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vcle{neon_type[0].no}" - doc: "Compare signed less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, uint64x1_t] - - [int64x2_t, uint64x2_t] - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vcle{neon_type[0].no}" - doc: "Floating-point compare less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vcle{type[0]}" - doc: "Floating-point compare less than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vcle_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[1]}", [b]] - - '0' - - - name: "vcge{neon_type[0].no}" - doc: "Compare signed greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, uint64x1_t] - - [int64x2_t, uint64x2_t] - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vcgez{neon_type[0].no}" - doc: "Compare signed greater than or equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: [simd_ge, [a, {FnCall: [transmute, [b]]}]] - - - name: "vcgezd_s64" - doc: "Compare signed greater than or equal to zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - vcgez_s64 - - - FnCall: [transmute, [a]] - - - name: "vclez{neon_type[0].no}" - doc: "Compare signed less than or equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmle]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_le - - - a - - FnCall: [transmute, [b]] - - - name: "vclez{neon_type[0].no}" - doc: "Floating-point compare less than or equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmle]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] - - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] - - [float64x1_t, uint64x1_t, f64, '0.0'] - - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_le - - - a - - FnCall: [transmute, [b]] - - - name: "vclez{type[0]}" - doc: "Floating-point compare less than or equal to zero" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vclez_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - '0' - - - name: "vcltz{neon_type[0].no}" - doc: "Compare signed less than zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmlt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_lt - - - a - - FnCall: [transmute, [b]] - - - name: "vcltz{neon_type[0].no}" - doc: "Floating-point compare less than zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmlt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] - - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] - - [float64x1_t, uint64x1_t, f64, '0.0'] - - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_lt - - - a - - FnCall: [transmute, [b]] - - - name: "vcltz{type[0]}" - doc: "Floating-point compare less than zero" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vcltz_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - '0' - - - name: "vcltzd_s64" - doc: "Compare less than zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [asr]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - vcltz_s64 - - - FnCall: [transmute, [a]] - - - name: "vcagt{neon_type[0].no}" - doc: "Floating-point absolute compare greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcagt{neon_type[0].no}" - links: - - link: "llvm.aarch64.neon.facgt.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcagt{type[0]}" - doc: "Floating-point absolute compare greater than" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32", i32] - - ["d_f64", "f64", "u64", i64] - compose: - - LLVMLink: - name: "vcagt{type[0]}" - links: - - link: "llvm.aarch64.neon.facgt.{type[3]}.{type[1]}" - arch: aarch64,arm64ec - - - name: "vcage{neon_type[0].no}" - doc: "Floating-point absolute compare greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcage{neon_type[0].no}" - links: - - link: "llvm.aarch64.neon.facge.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcage{type[0]}" - doc: "Floating-point absolute compare greater than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32", i32] - - ["d_f64", "f64", "u64", i64] - compose: - - LLVMLink: - name: "vcage{type[0]}" - links: - - link: "llvm.aarch64.neon.facge.{type[3]}.{type[1]}" - arch: aarch64,arm64ec - - - name: "vcalt{neon_type[0].no}" - doc: "Floating-point absolute compare less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: ["vcagt{neon_type[0].no}", [b, a]] - - - name: "vcalt{type[0]}" - doc: "Floating-point absolute compare less than" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: ["vcagt{type[0]}", [b, a]] - - - name: "vcale{neon_type[0].no}" - doc: "Floating-point absolute compare less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: ["vcage{neon_type[0].no}", [b, a]] - - - name: "vcale{type[0]}" - doc: "Floating-point absolute compare less than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: ["vcage{type[0]}", [b, a]] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int64x1_t, float64x1_t] - - [int64x2_t, float64x2_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt{type[0]}_{type[3]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "i32", "f32", s32] - - ["d_f64", "i64", "f64", s64] - compose: - - Identifier: ["a as {type[2]}", Symbol] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint64x1_t, float64x1_t] - - [uint64x2_t, float64x2_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt{type[2]}_{type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u32", "f32", "s_f32"] - - ["u64", "f64", "d_f64"] - compose: - - Identifier: ["a as {type[1]}", Symbol] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x1_t, float64x1_t] - - [int64x2_t, float64x2_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i32", "f32", 's_n_f32_s32', 'N >= 1 && N <= 32'] - - ["i64", "f64", 'd_n_f64_s64', 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: - - "a: {type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxs2fp.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint64x1_t, float64x1_t] - - [uint64x2_t, float64x2_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] - - - name: "vcvt{type[2]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["u32", "f32", 's_n_f32_u32', 'N >= 1 && N <= 32'] - - ["u64", "f64", 'd_n_f64_u64', 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: - - "a: {type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxu2fp.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", ["a.as_signed()", N]] - - - name: "vcvt{type[2]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "i32", "s_s32_f32", "32"] - - ["f64", "i64", "d_s64_f64", "64"] - compose: - - Identifier: ["a as i{type[3]}", Symbol] - - - name: "vcvt{type[2]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", "s_u32_f32"] - - ["f64", "u64", "d_u64_f64"] - compose: - - Identifier: ["a as {type[1]}", Symbol] - - - name: "vcvt_f64_f32" - doc: "Floating-point convert to higher precision long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, float64x2_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt_high_f64_f32" - doc: "Floating-point convert to higher precision long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x4_t, float64x2_t] - compose: - - Let: - - b - - float32x2_t - - FnCall: - - simd_shuffle! - - - a - - a - - '[2, 3]' - - FnCall: [simd_cast, [b]] - - - name: "vcvt_f32_f64" - doc: "Floating-point convert to lower precision narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x2_t, float32x2_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt_high_f32_f64" - doc: "Floating-point convert to lower precision narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, float64x2_t, float32x4_t] - compose: - - FnCall: - - simd_shuffle! - - - a - - FnCall: [simd_cast, [b]] - - '[0, 1, 2, 3]' - - - name: "vcvtx_f32_f64" - doc: "Floating-point convert to lower precision narrow, rounding to odd" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x2_t, float32x2_t] - compose: - - LLVMLink: - name: "vcvtx_f32_f64" - links: - - link: "llvm.aarch64.neon.fcvtxn.v2f32.v2f64" - arch: aarch64,arm64ec - - - name: "vcvtxd_f32_f64" - doc: "Floating-point convert to lower precision narrow, rounding to odd" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f64", "f32"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - vcvtx_f32_f64 - - - FnCall: [vdupq_n_f64, [a]] - - '0' - - - name: "vcvtx_high_f32_f64" - doc: "Floating-point convert to lower precision narrow, rounding to odd" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtxn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, float64x2_t, float32x4_t] - compose: - - FnCall: - - simd_shuffle! - - - a - - FnCall: [vcvtx_f32_f64, [b]] - - '[0, 1, 2, 3]' - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float64x1_t, int64x1_t, _n_s64_f64, '64'] - - [float64x2_t, int64x2_t, q_n_s64_f64, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.aarch64.neon.vcvtfp2fxs.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["f32", "i32", s_n_s32_f32, '32'] - - ["f64", "i64", d_n_s64_f64, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.aarch64.neon.vcvtfp2fxs.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t, _n_u64_f64, '64'] - - [float64x2_t, uint64x2_t, q_n_u64_f64, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.aarch64.neon.vcvtfp2fxu.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["f32", "u32", s_n_u32_f32, '32'] - - ["f64", "u64", d_n_u64_f64, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[3]}']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.aarch64.neon.vcvtfp2fxu.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvta{type[2]}" - doc: "Floating-point convert to signed integer, rounding to nearest with ties to away" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtas]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t, _s32_f32] - - [float32x4_t, int32x4_t, q_s32_f32] - - [float64x1_t, int64x1_t, _s64_f64] - - [float64x2_t, int64x2_t, q_s64_f64] - compose: - - LLVMLink: - name: "vcvta{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtas.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvta{type[2]}" - doc: "Floating-point convert to integer, rounding to nearest with ties to away" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtas]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "i32", 's_s32_f32'] - - ["f64", "i64", 'd_s64_f64'] - compose: - - LLVMLink: - name: "vcvta{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtas.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvta{type[2]}" - doc: "Floating-point convert to integer, rounding to nearest with ties to away" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtau]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", 's_u32_f32'] - - ["f64", "u64", 'd_u64_f64'] - compose: - - LLVMLink: - name: "vcvta{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtau.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to signed integer, rounding to nearest with ties to even" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtns]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t] - - [float32x4_t, int32x4_t] - - [float64x1_t, int64x1_t] - - [float64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtns.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtn{type[2]}" - doc: "Floating-point convert to signed integer, rounding to nearest with ties to even" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtns]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "i32", 's_s32_f32'] - - ["f64", "i64", 'd_s64_f64'] - compose: - - LLVMLink: - name: "vcvtn{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtns.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to signed integer, rounding toward minus infinity" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtms]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t] - - [float32x4_t, int32x4_t] - - [float64x1_t, int64x1_t] - - [float64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtms.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtm{type[2]}" - doc: "Floating-point convert to signed integer, rounding toward minus infinity" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtms]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "i32", 's_s32_f32'] - - ["f64", "i64", 'd_s64_f64'] - compose: - - LLVMLink: - name: "vcvtm{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtms.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtp{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to signed integer, rounding toward plus infinity" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtps]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t] - - [float32x4_t, int32x4_t] - - [float64x1_t, int64x1_t] - - [float64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vcvtp{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtps.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtp{type[2]}" - doc: "Floating-point convert to signed integer, rounding toward plus infinity" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtps]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "i32", 's_s32_f32'] - - ["f64", "i64", 'd_s64_f64'] - compose: - - LLVMLink: - name: "vcvtp{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtps.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to even" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtnu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcvtn{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtnu.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtn{type[2]}" - doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to even" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtnu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", 's_u32_f32'] - - ["f64", "u64", 'd_u64_f64'] - compose: - - LLVMLink: - name: "vcvtn{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtnu.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned integer, rounding toward minus infinity" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtmu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcvtm{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtmu.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtm{type[2]}" - doc: "Floating-point convert to unsigned integer, rounding toward minus infinity" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtmu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", s_u32_f32] - - ["f64", "u64", d_u64_f64] - compose: - - LLVMLink: - name: "vcvtm{type[2]}" - links: - - link: "llvm.aarch64.neon.fcvtmu.{type[1]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtp{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned integer, rounding toward plus infinity" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtpu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcvtp{neon_type[1].no}_{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.fcvtpu.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvtp{type[2]}" - doc: "Floating-point convert to unsigned integer, rounding toward plus infinity" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtpu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", s_u32_f32, 'i32'] - - ["f64", "u64", d_u64_f64, 'u64'] - compose: - - LLVMLink: - name: "vcvtp{type[2]}" - arguments: - - "a: {type[0]}" - return_type: "{type[1]}" - links: - - link: "llvm.aarch64.neon.fcvtpu.{type[3]}.{type[0]}" - arch: aarch64,arm64ec - - - name: "vdup{neon_type.laneq_nox}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [dup, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - poly64x2_t - - float64x2_t - compose: - - FnCall: [static_assert_uimm_bits!, [N, 1]] - - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] - - - name: "vdup{neon_type[1].lane_nox}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [dup, 'N = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [poly64x1_t, poly64x2_t] - - [float64x1_t, float64x2_t] - compose: - - FnCall: [static_assert!, ['N == 0']] - - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] - - - name: "vdup{neon_type.lane_nox}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - poly64x1_t - - float64x1_t - compose: - - FnCall: [static_assert!, ['N == 0']] - - Identifier: [a, Symbol] - - - name: "vdupd{neon_type[0].lane_nox}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x1_t, "i64"] - - [uint64x1_t, "u64"] - - [float64x1_t, "f64"] - compose: - - FnCall: [static_assert!, ['N == 0']] - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vdup_laneq_{neon_type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [poly64x2_t, poly64x1_t, 'u64'] - - [float64x2_t, float64x1_t, 'f64'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 1]] - - FnCall: - - "transmute::<{type[2]}, _>" - - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vdup{type[2]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, "i32", s_lane_s32] - - [int64x2_t, "i64", d_laneq_s64] - - [uint32x2_t, "u32", s_lane_u32] - - [uint64x2_t, "u64", d_laneq_u64] - - [float32x2_t, "f32", s_lane_f32] - - [float64x2_t, "f64", d_laneq_f64] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 1]] - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vdup{type[2]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 4']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, "i8", b_lane_s8] - - [int16x8_t, "i16", h_laneq_s16] - - [uint8x8_t, "u8", b_lane_u8] - - [uint16x8_t, "u16", h_laneq_u16] - - [poly8x8_t, "p8", b_lane_p8] - - [poly16x8_t, "p16", h_laneq_p16] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 3]] - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vdup{type[2]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 8']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x16_t, "i8", b_laneq_s8] - - [uint8x16_t, "u8", b_laneq_u8] - - [poly8x16_t, "p8", b_laneq_p8] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 4]] - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vdup{type[2]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", h_lane_s16] - - [int32x4_t, "i32", s_laneq_s32] - - [uint16x4_t, "u16", h_lane_u16] - - [uint32x4_t, "u32", s_laneq_u32] - - [poly16x4_t, "p16", h_lane_p16] - - [float32x4_t, "f32", s_laneq_f32] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 2]] - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ext, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [poly64x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - - [float64x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vmla{neon_type.no}" - doc: "Floating-point multiply-add to accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] - - - name: "vmlal_high_{neon_type[1]}" - doc: "Signed multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]', '[4, 5, 6, 7]'] - - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]', '[2, 3]'] - compose: - - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [c, c, "{type[4]}"]]}] - - FnCall: ["vmlal_{neon_type[2]}", [a, b, c]] - - - name: "vmlal_high_{neon_type[1]}" - doc: "Unsigned multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] - compose: - - Let: - - b - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] - - Let: - - c - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - FnCall: ["vmlal_{neon_type[1]}", [a, b, c]] - - - name: "vmlsl_high_{neon_type[1]}" - doc: "Signed multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]'] - - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]'] - compose: - - Let: - - b - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] - - Let: - - c - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - FnCall: ["vmlsl_{neon_type[1]}", [a, b, c]] - - - name: "vmlsl_high_{neon_type[1]}" - doc: "Unsigned multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] - compose: - - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [c, c, "{type[3]}"]]}] - - FnCall: ["vmlsl_{neon_type[1]}", [a, b, c]] - - - name: "vmovn_high{neon_type[1].noq}" - doc: Extract narrow - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: [*neon-stable] - assert_instr: [xtn2] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] - - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] - compose: - - Let: - - c - - "{neon_type[0]}" - - FnCall: - - simd_cast - - - b - - FnCall: - - simd_shuffle! - - - a - - c - - "{type[3]}" - - - name: "vneg{neon_type.no}" - doc: Negate - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [neg] - safety: - unsafe: [neon] - types: - - int64x1_t - - int64x2_t - compose: - - FnCall: - - simd_neg - - - a - - - name: "vnegd_s64" - doc: Negate - arguments: ["a: {type}"] - return_type: "{type}" - attr: [*neon-stable] - assert_instr: [neg] - safety: - unsafe: [neon] - types: - - i64 - compose: - - MethodCall: [a, wrapping_neg, []] - - - name: "vneg{neon_type.no}" - doc: Negate - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - assert_instr: [fneg] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: - - simd_neg - - - a - - - name: "vqneg{type[1]}" - doc: Signed saturating negate - arguments: ["a: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqneg] - safety: - unsafe: [neon] - types: - - [i8, 'b_s8', 's8'] - - [i16, 'h_s16', 's16'] - - [i32, 's_s32', 's32'] - - [i64, 'd_s64', 's64'] - compose: - - FnCall: - - 'simd_extract!' - - - FnCall: - - 'vqneg_{type[2]}' - - - FnCall: ['vdup_n_{type[2]}', [a]] - - 0 - - - name: "vqneg{neon_type[0].no}" - doc: Signed saturating negate - arguments: ["a: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqneg] - safety: - unsafe: [neon] - types: - - [int64x1_t, 'i64'] - - [int64x2_t, 'i64'] - compose: - - LLVMLink: - name: "sqneg.{neon_type[0].no}" - links: - - link: "llvm.aarch64.neon.sqneg.v{neon_type[0].lane}{type[1]}" - arch: aarch64,arm64ec - - - name: "vqsub{type[1]}" - doc: Saturating subtract - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqsub] - safety: - unsafe: [neon] - types: - - [i32, 's_s32', 'i32'] - - [i64, 'd_s64', 'i64'] - compose: - - LLVMLink: - name: "sqsub.{type[0]}" - links: - - link: "llvm.aarch64.neon.sqsub.{type[2]}" - arch: aarch64,arm64ec - - - name: "vqsub{type[1]}" - doc: Saturating subtract - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [uqsub] - safety: - unsafe: [neon] - types: - - [u32, 's_u32', 'i32'] - - [u64, 'd_u64', 'i64'] - compose: - - LLVMLink: - name: "uqsub.{type[0]}" - links: - - link: "llvm.aarch64.neon.uqsub.{type[2]}" - arch: aarch64,arm64ec - - - name: "vqsub{type[3]}" - doc: Saturating subtract - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqsub] - safety: - unsafe: [neon] - types: - - [i8, int8x8_t, s8, 'b_s8'] - - [i16, int16x4_t, s16, 'h_s16'] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[2]}" - - - a - - Let: - - b - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[2]}" - - - b - - FnCall: - - 'simd_extract!' - - - FnCall: - - "vqsub_{type[2]}" - - - a - - b - - "0" - - - name: "vqsub{type[3]}" - doc: Saturating subtract - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [uqsub] - safety: - unsafe: [neon] - types: - - [u8, uint8x8_t, u8, 'b_u8'] - - [u16, uint16x4_t, u16, 'h_u16'] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[2]}" - - - a - - Let: - - b - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[2]}" - - - b - - FnCall: - - 'simd_extract!' - - - FnCall: - - "vqsub_{type[2]}" - - - a - - b - - "0" - - - name: "vrbit{neon_type.no}" - doc: Reverse bit order - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [rbit] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - compose: - - LLVMLink: - name: "rbit.{neon_type}" - links: - - link: "llvm.aarch64.neon.rbit.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrbit{neon_type[0].no}" - doc: Reverse bit order - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: [*neon-stable] - assert_instr: [rbit] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [poly8x8_t, int8x8_t] - - [poly8x16_t, int8x16_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vrbit{neon_type[1].no}" - - - FnCall: [transmute, [a]] - - - name: "vrndx{neon_type.no}" - doc: "Floating-point round to integral exact, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frintx] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.rint.{neon_type}" - links: - - link: "llvm.rint.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnda{neon_type.no}" - doc: "Floating-point round to integral, to nearest with ties to away" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frinta] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.round.{neon_type}" - links: - - link: "llvm.round.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrndn{neon_type.no}" - doc: "Floating-point round to integral, to nearest with ties to even" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frintn] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "frintn.{neon_type}" - links: - - link: "llvm.aarch64.neon.frintn.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrndns_{type}" - doc: "Floating-point round to integral, to nearest with ties to even" - arguments: ["a: {type}"] - return_type: "{type}" - attr: [*neon-stable] - assert_instr: [frintn] - safety: - unsafe: [neon] - types: - - f32 - compose: - - LLVMLink: - name: "roundeven.{type}" - links: - - link: "llvm.roundeven.{type}" - arch: aarch64,arm64ec - - - name: "vrndm{neon_type.no}" - doc: "Floating-point round to integral, toward minus infinity" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frintm] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.floor.{neon_type}" - links: - - link: "llvm.floor.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrndp{neon_type.no}" - doc: "Floating-point round to integral, toward plus infinity" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frintp] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.ceil.{neon_type}" - links: - - link: "llvm.ceil.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnd{neon_type.no}" - doc: "Floating-point round to integral, toward zero" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frintz] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.trunc.{neon_type}" - links: - - link: "llvm.trunc.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrndi{neon_type.no}" - doc: "Floating-point round to integral, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [frinti] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.nearbyint.{neon_type}" - links: - - link: "llvm.nearbyint.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqadd{type[1]}" - doc: Saturating add - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [uqadd] - safety: - unsafe: [neon] - types: - - [u32, 's_u32', i32] - - [u64, 'd_u64', i64] - compose: - - LLVMLink: - name: "uqadd.{type[2]}" - links: - - link: "llvm.aarch64.neon.uqadd.{type[2]}" - arch: aarch64,arm64ec - - - name: "vqadd{type[1]}" - doc: Saturating add - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqadd] - safety: - unsafe: [neon] - types: - - [i32, 's_s32', i32] - - [i64, 'd_s64', i64] - compose: - - LLVMLink: - name: "uqadd.{type[2]}" - links: - - link: "llvm.aarch64.neon.sqadd.{type[2]}" - arch: aarch64,arm64ec - - - name: "vqadd{type[2]}" - doc: Saturating add - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [sqadd] - safety: - unsafe: [neon] - types: - - [i8, int8x8_t, 'b_s8'] - - [i16, int16x4_t, 'h_s16'] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[0]}" - - - a - - Let: - - b - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[0]}" - - - b - - FnCall: - - simd_extract! - - - FnCall: - - "vqadd_{type[0]}" - - - a - - b - - "0" - - - name: "vqadd{type[2]}" - doc: Saturating add - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: [*neon-stable] - assert_instr: [uqadd] - safety: - unsafe: [neon] - types: - - [u8, uint8x8_t, 'b_u8'] - - [u16, uint16x4_t, 'h_u16'] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[0]}" - - - a - - Let: - - b - - "{neon_type[1]}" - - FnCall: - - "vdup_n_{type[0]}" - - - b - - FnCall: - - simd_extract! - - - FnCall: - - "vqadd_{type[0]}" - - - a - - b - - "0" - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld1] - safety: - unsafe: [neon] - types: - - ["*const f64", float64x1x2_t] - - ["*const f64", float64x2x2_t] - - ["*const f64", float64x1x3_t] - - ["*const f64", float64x2x3_t] - - ["*const f64", float64x1x4_t] - - ["*const f64", float64x2x4_t] - compose: - - LLVMLink: - name: "vld1{neon_type[1].no}" - links: - - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f64" - arch: aarch64,arm64ec - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const i8", int8x16x2_t, i8, int8x16_t, "4"] - - ["*const i64", int64x2x2_t, i64, int64x2_t, "1"] - - ["*const f64", float64x2x2_t, f64, float64x2_t, "1"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[4]}" - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "a: {neon_type[3]}" - - "b: {neon_type[3]}" - - "n: i64" - - "ptr: *const i8" - links: - - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" - arch: aarch64,arm64ec - - FnCall: ["_vld2{neon_type[1].lane_nox}", ["b.0", "b.1", "LANE as i64", "a as _"]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x2_t, i64, int64x1_t] - - ["*const f64", float64x1x2_t, f64, float64x1_t] - compose: - - FnCall: ["static_assert!", ['LANE == 0']] - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "a: {neon_type[3]}" - - "b: {neon_type[3]}" - - "n: i64" - - "ptr: *const i8" - links: - - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" - arch: aarch64,arm64ec - - FnCall: ["_vld2{neon_type[1].lane_nox}", ["b.0", "b.1", "LANE as i64", "a as _"]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - "static_assert!" - - - 'LANE == 0' - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - "static_assert!" - - - 'LANE == 0' - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x16x2_t, int8x16x2_t, "4"] - - ["*const p8", poly8x16x2_t, int8x16x2_t, "4"] - - ["*const u64", uint64x2x2_t, int64x2x2_t, "1"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[3]}" - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x2_t, int64x2x2_t, "1"] - compose: - - FnCall: ["static_assert_uimm_bits!", [LANE, '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld2] - safety: - unsafe: [neon] - types: - - ["*const f64", float64x2x2_t, f64, float64x2_t] - - ["*const i64", int64x2x2_t, i64, int64x2_t] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const {neon_type[3]}" - links: - - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as _" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["*const f64", float64x1x2_t, f64, float64x1_t] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const {neon_type[3]}" - links: - - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as _" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld2] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-stable - assert_instr: [ld2] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-stable - assert_instr: [ld2r] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x2x2_t, i64] - - ["*const f64", float64x1x2_t, f64] - - ["*const f64", float64x2x2_t, f64] - compose: - - LLVMLink: - name: "vld2dup.{neon_type[1]}" - arguments: - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].dup_nox}" - - - "a as _" - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-stable - assert_instr: [ld2r] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-aes - - *neon-stable - assert_instr: [ld2r] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].lane_nox}" - doc: "Load multiple 3-element structures to two registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i8', int8x16x3_t, int8x16_t, i8, '3'] - - ['*const i64', int64x2x3_t, int64x2_t, i64, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - LLVMLink: - name: 'ld3lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: "Load multiple 3-element structures to three registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const f64', float64x2x3_t, float64x2_t, f64, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - LLVMLink: - name: 'ld3lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const f64', float64x1x3_t, float64x1_t, f64] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'vld3.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: "Load multiple 3-element structures to two registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i64', int64x1x3_t, int64x1_t, i64] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'vld3.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const p8', poly8x16x3_t, int8x16x3_t, '4'] - - ['*const u8', uint8x16x3_t, int8x16x3_t, '4'] - - ['*const u64', uint64x2x3_t, int64x2x3_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const u64', uint64x1x3_t, int64x1x3_t, '1'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x2x3_t, int64x2x3_t] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', 1]] - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x1x3_t, int64x1x3_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - safety: - unsafe: [neon] - assert_instr: [ld3] - types: - - ['*const i64', int64x2x3_t, '*const int64x2_t', i64] - - ['*const f64', float64x2x3_t, '*const float64x2_t', f64] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].nox}' - arguments: - - 'ptr: {type[2]}' - links: - - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - safety: - unsafe: [neon] - assert_instr: [nop] - types: - - ['*const f64', float64x1x3_t, '*const float64x1_t', f64] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].nox}' - arguments: - - 'ptr: {type[2]}' - links: - - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - safety: - unsafe: [neon] - assert_instr: [ld3] - types: - - ['*const u64', uint64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-stable - safety: - unsafe: [neon] - assert_instr: [ld3] - types: - - ['*const p64', poly64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld3r] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x2x3_t, i64] - - ["*const f64", float64x1x3_t, f64] - - ["*const f64", float64x2x3_t, f64] - compose: - - LLVMLink: - name: 'ld3r{neon_type[1].dup_nox}' - arguments: - - 'ptr: {type[0]}' - links: - - link: 'llvm.aarch64.neon.ld3r.v{neon_type[1].lane}{type[2]}.p0{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as _']] - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld3r] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld3{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-stable - assert_instr: [ld3r] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld3{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-stable - assert_instr: [ld4] - safety: - unsafe: [neon] - types: - - ['*const f64', float64x2x4_t, f64, '*const float64x2_t'] - - ['*const i64', int64x2x4_t, i64, '*const int64x2_t'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: {type[3]}' - links: - - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-stable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ['*const f64', float64x1x4_t, f64, '*const float64x1_t'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: {type[3]}' - links: - - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ld4] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-stable - - *neon-aes - assert_instr: [ld4] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i8', int8x16x4_t, int8x16_t, i8, '3'] - - ['*const i64', int64x2x4_t, int64x2_t, i64, '1'] - - ['*const f64', float64x2x4_t, float64x2_t, f64, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - LLVMLink: - name: 'ld4lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i64', int64x1x4_t, int64x1_t, i64] - - ['*const f64', float64x1x4_t, float64x1_t, f64] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'ld4lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const p8', poly8x16x4_t, int8x16x4_t, '4'] - - ['*const u8', uint8x16x4_t, int8x16x4_t, '4'] - - ['*const u64', uint64x2x4_t, int64x2x4_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x2x4_t, int64x2x4_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const u64', uint64x1x4_t, int64x1x4_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: - - target_feature - - - 'enable = "neon,aes"' - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x1x4_t, int64x1x4_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - types: - - ['*mut f64', float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - types: - - ['*mut f64', float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *neon-stable - assert_instr: [st1] - types: - - ['f64', float64x1x2_t, float64x1_t] - compose: - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *neon-stable - assert_instr: [st2] - types: - - [i64, int64x2x2_t, int64x2_t] - - [f64, float64x2x2_t, float64x2_t] - compose: - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [i64, int64x1x2_t, int64x1_t] - - [f64, float64x1x2_t, float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [i8, int8x16x2_t, int8x16_t, '4'] - - [i64, int64x2x2_t, int64x2_t, '1'] - - [f64, float64x2x2_t, float64x2_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [u8, uint8x16x2_t, int8x16x2_t, '4'] - - [u64, uint64x2x2_t, int64x2x2_t, '1'] - - [p8, poly8x16x2_t, int8x16x2_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] - - FnCall: - - "vst2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [u64, uint64x1x2_t, int64x1x2_t, '1'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - assert_instr: [st2] - safety: - unsafe: [neon] - types: - - [u64, uint64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - "vst2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [p64, poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [p64, poly64x2x2_t, int64x2x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '1']] - - FnCall: - - "vst2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2]]}]] - - *neon-stable - safety: - unsafe: [neon] - types: - - [p64, poly64x2x2_t, int64x2x2_t] - compose: - - FnCall: - - "vst2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [f64, float64x1x3_t, float64x1_t] - compose: - - LLVMLink: - name: 'st3.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [p64, poly64x1x3_t, int64x1x3_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst3{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [p64, poly64x2x3_t, int64x2x3_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: - - "vst3{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - assert_instr: [st3] - safety: - unsafe: [neon] - types: - - [p64, poly64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - "vst3{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [st3] - safety: - unsafe: [neon] - types: - - [i64, int64x2x3_t, int64x2_t] - - [f64, float64x2x3_t, float64x2_t] - compose: - - LLVMLink: - name: 'st3.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [st3] - safety: - unsafe: [neon] - types: - - [u64, uint64x2x3_t, int64x2x3_t] - compose: - - FnCall: - - "vst3{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [u64, uint64x1x3_t, int64x1x3_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst3{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [u8, uint8x16x3_t, int8x16x3_t, '4'] - - [u64, uint64x2x3_t, int64x2x3_t, '1'] - - [p8, poly8x16x3_t, int8x16x3_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vst3{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [f64, float64x2x3_t, float64x2_t, '1'] - - [i8, int8x16x3_t, int8x16_t, '4'] - - [i64, int64x2x3_t, int64x2_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - LLVMLink: - name: 'st3lane.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i64, int64x1x3_t, int64x1_t, '1'] - - [f64, float64x1x3_t, float64x1_t, '1'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'st3lane.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [f64, float64x1x4_t, float64x1_t] - compose: - - LLVMLink: - name: 'st4.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [p64, poly64x1x4_t, int64x1x4_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst4{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [p64, poly64x2x4_t, int64x2x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: - - "vst4{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - *neon-aes - assert_instr: [st4] - safety: - unsafe: [neon] - types: - - [p64, poly64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - "vst4{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [st4] - safety: - unsafe: [neon] - types: - - [i64, int64x2x4_t, int64x2_t] - - [f64, float64x2x4_t, float64x2_t] - compose: - - LLVMLink: - name: 'st4.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*neon-stable] - assert_instr: [st4] - safety: - unsafe: [neon] - types: - - [u64, uint64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - "vst4{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [u64, uint64x1x4_t, int64x1x4_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vst4{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-stable - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [u8, uint8x16x4_t, int8x16x4_t, '4'] - - [u64, uint64x2x4_t, int64x2x4_t, '1'] - - [p8, poly8x16x4_t, int8x16x4_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vst4{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [f64, float64x2x4_t, float64x2_t, '1'] - - [i8, int8x16x4_t, int8x16_t, '4'] - - [i64, int64x2x4_t, int64x2_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - LLVMLink: - name: 'st4lane.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i64, int64x1x4_t, int64x1_t, '1'] - - [f64, float64x1x4_t, float64x1_t, '1'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - LLVMLink: - name: 'st4lane.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vusdot{neon_type[0].laneq_nox}" - doc: "Dot product index form with unsigned and signed integers" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-i8mm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usdot, 'LANE = 3']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [int32x2_t, uint8x8_t, int8x16_t, '[LANE as u32, LANE as u32]'] - - [int32x4_t, uint8x16_t, int8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '2']] - - Let: [c, int32x4_t, {FnCall: [transmute, [c]]}] - - Let: [c, "{neon_type[0]}", {FnCall: [simd_shuffle!, [c, c, "{type[3]}"]]}] - - FnCall: ["vusdot{neon_type[0].no}", [a, b, {FnCall: [transmute, [c]]}]] - - - name: "vsudot{neon_type[0].laneq_nox}" - doc: "Dot product index form with signed and unsigned integers" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-i8mm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sudot, 'LANE = 3']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [int32x2_t, int8x8_t, uint8x16_t, '[LANE as u32, LANE as u32]', uint32x2_t] - - [int32x4_t, int8x16_t, uint8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]', uint32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, 2]] - - Let: - - c - - uint32x4_t - - FnCall: [transmute, [c]] - - Let: - - c - - "{type[4]}" - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - FnCall: ["vusdot{neon_type[0].no}", [a, {FnCall: [transmute, [c]]}, b]] - - - name: "vmul{neon_type.no}" - doc: Multiply - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fmul] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_mul, [a, b]] - - - name: "vmull_high{neon_type[0].noq}" - doc: Signed multiply long - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[3]}" - attr: [*neon-stable] - assert_instr: [smull2] - safety: - unsafe: [neon] - types: - - [int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int16x8_t] - - [int16x8_t, int16x4_t, '[4, 5, 6, 7]', int32x4_t] - - [int32x4_t, int32x2_t, '[2, 3]', int64x2_t] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] - - Let: - - b - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] - - FnCall: ["vmull_{neon_type[0]}", [a, b]] - - - name: "vmull_high{neon_type[0].noq}" - doc: "Unsigned multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[3]}" - attr: [*neon-stable] - assert_instr: [umull2] - safety: - unsafe: [neon] - types: - - [uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', uint16x8_t] - - [uint16x8_t, uint16x4_t, '[4, 5, 6, 7]', uint32x4_t] - - [uint32x4_t, uint32x2_t, '[2, 3]', uint64x2_t] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] - - Let: - - b - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] - - FnCall: ["vmull_{neon_type[0]}", [a, b]] - - - name: "vmull_p64" - doc: "Polynomial multiply long" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-aes - - *neon-stable - safety: - unsafe: [neon] - assert_instr: [pmull] - types: - - ["p64", "p128"] - compose: - - LLVMLink: - name: "pmull.{type[0]}" - return_type: "int8x16_t" - links: - - link: "llvm.aarch64.neon.pmull64" - arch: aarch64,arm64ec - - FnCall: [transmute, [{FnCall: ["_vmull_p64", [a, b]]}]] - - - name: "vmull_high{neon_type[0].noq}" - doc: "Polynomial multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[3]}" - attr: - - *neon-stable - safety: - unsafe: [neon] - assert_instr: [pmull] - types: - - [poly8x16_t, poly8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', poly16x8_t] - compose: - - Let: - - a - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] - - Let: - - b - - "{neon_type[1]}" - - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] - - FnCall: ["vmull_{neon_type[0]}", [a, b]] - - - name: "vmull_high{neon_type[0].noq}" - doc: "Polynomial multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-aes - - *neon-stable - safety: - unsafe: [neon] - assert_instr: [pmull] - types: - - [poly64x2_t, "p128"] - compose: - - FnCall: - - "vmull_{neon_type[0]}" - - - FnCall: [simd_extract!, [a, '1']] - - FnCall: [simd_extract!, [b, '1']] - - - name: "vmulx{neon_type.no}" - doc: Floating-point multiply extended - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fmulx] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fmulx.{neon_type.no}" - links: - - link: "llvm.aarch64.neon.fmulx.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmulx{type[0]}" - doc: Floating-point multiply extended - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [fmulx] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32"] - - ["d_f64", "f64"] - compose: - - LLVMLink: - name: "fmulx.{type[1]}" - links: - - link: "llvm.aarch64.neon.fmulx.{type[1]}" - arch: aarch64,arm64ec - - - name: "vmulx_lane_f64" - doc: Floating-point multiply extended - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - vmulx_f64 - - - a - - FnCall: - - 'transmute::' - - - FnCall: - - "simd_extract!" - - - b - - 'LANE as u32' - - - name: "vmulx{type[0]}" - doc: Floating-point multiply extended - arguments: ["a: {type[1]}", "b: {neon_type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ["q_lane_f64", float64x2_t, float64x1_t, "q_f64", '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vmulx{type[3]}" - - - a - - FnCall: - - "simd_shuffle!" - - - b - - b - - "{type[4]}" - - - name: "vmulx{type[0]}" - doc: Floating-point multiply extended - arguments: ["a: {type[1]}", "b: {neon_type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ["d_lane_f64", "f64", float64x1_t, "d_f64", 'LANE as u32'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - "vmulx{type[3]}" - - - a - - FnCall: - - "simd_extract!" - - - b - - "{type[4]}" - - - name: "vmulx_laneq_f64" - doc: Floating-point multiply extended - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float64x1_t, float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '1']] - - FnCall: - - vmulx_f64 - - - a - - FnCall: - - 'transmute::' - - - FnCall: - - "simd_extract!" - - - b - - 'LANE as u32' - - - name: "vmulx{type[0]}" - doc: Floating-point multiply extended - arguments: ["a: {type[1]}", "b: {neon_type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['_lane_f32', float32x2_t, float32x2_t, '1', '_f32', '[LANE as u32, LANE as u32]'] - - ['_laneq_f32', float32x2_t, float32x4_t, '2', '_f32', '[LANE as u32, LANE as u32]'] - - ['q_lane_f32', float32x4_t, float32x2_t, '1', 'q_f32', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - ['q_laneq_f32', float32x4_t, float32x4_t, '2', 'q_f32', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - ['q_laneq_f64', float64x2_t, float64x2_t, '1', 'q_f64', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] - - FnCall: - - "vmulx{type[4]}" - - - a - - FnCall: - - "simd_shuffle!" - - - b - - b - - "{type[5]}" - - - name: "vmulx{type[0]}" - doc: Floating-point multiply extended - arguments: ["a: {type[1]}", "b: {neon_type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmulx, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['s_lane_f32', f32, float32x2_t, '1', 's_f32', 'LANE as u32'] - - ['s_laneq_f32', f32, float32x4_t, '2', 's_f32', 'LANE as u32'] - - ['d_laneq_f64', f64, float64x2_t, '1', 'd_f64', 'LANE as u32'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] - - FnCall: - - "vmulx{type[4]}" - - - a - - FnCall: - - "simd_extract!" - - - b - - "{type[5]}" - - - name: "vfma{neon_type.no}" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-stable - assert_instr: [fmadd] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - LLVMLink: - name: "_vfma{neon_type.no}" - arguments: ["a: {type}", "b: {type}", "c: {type}"] - links: - - link: "llvm.fma.{neon_type}" - arch: aarch64,arm64ec - - FnCall: ["_vfma{neon_type.no}", [b, c, a]] - - - name: "vfma{neon_type.no}" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-stable - assert_instr: [fmla] - safety: - unsafe: [neon] - types: - - float64x2_t - compose: - - LLVMLink: - name: "_vfma{neon_type.no}" - arguments: ["a: {type}", "b: {type}", "c: {type}"] - links: - - link: "llvm.fma.{neon_type}" - arch: aarch64,arm64ec - - FnCall: ["_vfma{neon_type.no}", [b, c, a]] - - - name: "vfma_n_f64" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - assert_instr: [fmadd] - safety: - unsafe: [neon] - types: - - [float64x1_t, f64] - compose: - - FnCall: - - "vfma_f64" - - - a - - b - - FnCall: - - "vdup_n_f64" - - - c - - - name: "vfmaq_n_f64" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - assert_instr: [fmla] - safety: - unsafe: [neon] - types: - - [float64x2_t, f64] - compose: - - FnCall: - - "vfmaq_f64" - - - a - - b - - FnCall: - - "vdupq_n_f64" - - - c - - - name: "vdiv{neon_type.no}" - doc: "Divide" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fdiv] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_div, [a, b]] - - - name: "vsub{neon_type.no}" - doc: "Subtract" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fsub] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_sub, [a, b]] - - - name: "vsub{type[0]}" - doc: "Subtract" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ['d_s64', 'i64'] - - ['d_u64', 'u64'] - compose: - - MethodCall: [a, wrapping_sub, [b]] - - - name: "vaddv{neon_type[0].no}" - doc: Floating-point add across vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - assert_instr: [faddp] - safety: - unsafe: [neon] - types: - - [float32x2_t, f32] - - [float32x4_t, f32] - - [float64x2_t, f64] - compose: - - LLVMLink: - name: "faddv.{type[1]}.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.faddv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vaddlv{neon_type[0].no}" - doc: Signed Add Long across Vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [saddlv] - safety: - unsafe: [neon] - types: - - [int16x4_t, i32] - - [int16x8_t, i32] - - [int32x4_t, i64] - compose: - - LLVMLink: - name: "llvm.aarch64.neon.saddlv.{type[1]}.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.saddlv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vaddlv{neon_type.no}" - doc: Signed Add Long across Vector - arguments: ["a: {neon_type}"] - return_type: "i64" - attr: [*neon-stable] - assert_instr: [saddlp] - safety: - unsafe: [neon] - types: - - int32x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.saddlv.i64.v2i32" - links: - - link: "llvm.aarch64.neon.saddlv.i64.v2i32" - arch: aarch64,arm64ec - - - name: "vaddlv{neon_type[0].no}" - doc: Unsigned Add Long across Vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [uaddlv] - safety: - unsafe: [neon] - types: - - [uint16x4_t, u32, i32] - - [uint16x8_t, u32, i32] - - [uint32x4_t, u64, i64] - compose: - - LLVMLink: - name: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vaddlv{neon_type[0].no}" - doc: Unsigned Add Long across Vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [uaddlp] - safety: - unsafe: [neon] - types: - - [uint32x2_t, u64, i64] - compose: - - LLVMLink: - name: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.uaddlv.{type[2]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vsubw_high{neon_type[1].noq}" - doc: Signed Subtract Wide - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: [*neon-stable] - assert_instr: [ssubw] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x16_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [int32x4_t, int16x8_t, int16x4_t, '[4, 5, 6, 7]'] - - [int64x2_t, int32x4_t, int32x2_t, '[2, 3]'] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] - - FnCall: - - simd_sub - - - a - - FnCall: [simd_cast, [c]] - - - name: "vsubw_high{neon_type[1].noq}" - doc: Unsigned Subtract Wide - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: [*neon-stable] - assert_instr: [usubw] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]'] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] - - FnCall: - - simd_sub - - - a - - FnCall: [simd_cast, [c]] - - - name: "vsubl_high{neon_type[0].noq}" - doc: "Signed Subtract Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [ssubl] - safety: - unsafe: [neon] - types: - - [int8x16_t, int16x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int8x8_t] - - [int16x8_t, int32x4_t, '[4, 5, 6, 7]', int16x4_t] - - [int32x4_t, int64x2_t, '[2, 3]', int32x2_t] - compose: - - Let: - - c - - "{neon_type[3]}" - - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] - - Let: - - d - - "{neon_type[1]}" - - FnCall: [simd_cast, [c]] - - Let: - - e - - "{neon_type[3]}" - - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] - - Let: - - f - - "{neon_type[1]}" - - FnCall: [simd_cast, [e]] - - FnCall: [simd_sub, [d, f]] - - - name: "vsubl_high{neon_type[0].noq}" - doc: "Unsigned Subtract Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [usubl] - safety: - unsafe: [neon] - types: - - [uint8x16_t, uint16x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', uint8x8_t] - - [uint16x8_t, uint32x4_t, '[4, 5, 6, 7]', uint16x4_t] - - [uint32x4_t, uint64x2_t, '[2, 3]', uint32x2_t] - compose: - - Let: - - c - - "{neon_type[3]}" - - FnCall: [simd_shuffle!, [a, a, "{type[2]}"]] - - Let: - - d - - "{neon_type[1]}" - - FnCall: [simd_cast, [c]] - - Let: - - e - - "{neon_type[3]}" - - FnCall: [simd_shuffle!, [b, b, "{type[2]}"]] - - Let: - - f - - "{neon_type[1]}" - - FnCall: [simd_cast, [e]] - - FnCall: [simd_sub, [d, f]] - - - name: "vbcax{neon_type.no}" - doc: Bit clear and exclusive OR - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - assert_instr: [bcax] - safety: - unsafe: [neon] - types: - - int8x16_t - - int16x8_t - - int32x4_t - - int64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.crypto.bcaxs.{neon_type}" - links: - - link: "llvm.aarch64.crypto.bcaxs.{neon_type}" - arch: aarch64,arm64ec - - - name: "vbcax{neon_type.no}" - doc: Bit clear and exclusive OR - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - assert_instr: [bcax] - safety: - unsafe: [neon] - types: - - uint8x16_t - - uint16x8_t - - uint32x4_t - - uint64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.crypto.bcaxu.{neon_type}" - links: - - link: "llvm.aarch64.crypto.bcaxu.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcadd{neon_type.rot270}" - doc: "Floating-point complex add" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcadd] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcadd.rot270.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcadd.rot270.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcadd{neon_type.rot90}" - doc: "Floating-point complex add" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcadd] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcadd.rot90.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcadd.rot90.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcmla{neon_type.no}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcmla] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcmla.rot0.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcmla.rot0.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcmla{neon_type.rot90}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcmla] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcmla.rot90.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcmla.rot90.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcmla{neon_type.rot270}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcmla] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcmla.rot270.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcmla.rot270.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcmla{neon_type[0].laneq_nox}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, 1]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].no}", [a, b, c]] - - - name: "vcmla{neon_type[0].rot90_laneq}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, 1]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].rot90}", [a, b, c]] - - - name: "vcmla{neon_type[0].rot90_lane}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].rot90}", [a, b, c]] - - - name: "vcmla{neon_type.rot180}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - *neon-unstable-fcma - assert_instr: [fcmla] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "llvm.aarch64.neon.vcmla.rot180.{neon_type}" - links: - - link: "llvm.aarch64.neon.vcmla.rot180.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcmla{neon_type[0].rot180_laneq}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, 1]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].rot180}", [a, b, c]] - - - name: "vcmla{type[3]}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]', '_rot180_lane_f32'] - - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]', 'q_rot180_lane_f32'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].rot180}", [a, b, c]] - - - name: "vcmla{neon_type[0].rot270_laneq}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x4_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, 1]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].rot270}", [a, b, c]] - - - name: "vcmla{neon_type[0].lane_nox}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, "{type[2]}"]] - - FnCall: ["vcmla{neon_type[0].no}", [a, b, c]] - - - name: "vcmla{neon_type[0].rot270_lane}" - doc: Floating-point complex multiply accumulate - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [target_feature, ['enable = "neon,fcma"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-fcma - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1]'] - - [float32x4_t, float32x2_t, '[2 * LANE as u32, 2 * LANE as u32 + 1, 2 * LANE as u32, 2 * LANE as u32 + 1]'] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Let: [c, "{neon_type[0]}", {FnCall: [simd_shuffle!, [c, c, "{type[2]}"]]}] - - FnCall: ["vcmla{neon_type[0].rot270}", [a, b, c]] - - - name: "vdot{neon_type[0].laneq_nox}" - doc: Dot product arithmetic (indexed) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - static_defs: ["const LANE: i32"] - attr: - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sdot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']] - safety: - unsafe: [neon] - types: - - [int32x2_t, int8x8_t, int8x16_t, int32x4_t, '[LANE as u32, LANE as u32]'] - - [int32x4_t, int8x16_t, int8x16_t, int32x4_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '2']] - - Let: - - c - - "{neon_type[3]}" - - FnCall: [transmute, [c]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] - - FnCall: - - "vdot{neon_type[0].no}" - - - a - - b - - FnCall: [transmute, [c]] - - - name: "vdot{neon_type[0].laneq_nox}" - doc: Dot product arithmetic (indexed) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - static_defs: ["const LANE: i32"] - attr: - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [udot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']] - safety: - unsafe: [neon] - types: - - [uint32x2_t, uint8x8_t, uint8x16_t, uint32x4_t, '[LANE as u32, LANE as u32]'] - - [uint32x4_t, uint8x16_t, uint8x16_t, uint32x4_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '2']] - - Let: - - c - - "{neon_type[3]}" - - FnCall: [transmute, [c]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] - - FnCall: - - "vdot{neon_type[0].no}" - - - a - - b - - FnCall: [transmute, [c]] - - - name: "vmax{neon_type.no}" - doc: Maximum (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fmax] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fmax.{neon_type}" - links: - - link: "llvm.aarch64.neon.fmax.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmaxnm{neon_type.no}" - doc: Floating-point Maximum Number (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fmaxnm] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fmaxnm.{neon_type}" - links: - - link: "llvm.aarch64.neon.fmaxnm.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmaxnmv{neon_type[0].no}" - doc: Floating-point maximum number across vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [fmaxnmp] - safety: - unsafe: [neon] - types: - - [float32x2_t, f32] - - [float64x2_t, f64] - compose: - - LLVMLink: - name: "fmaxnmv.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fmaxnmv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vmaxnmv{neon_type[0].no}" - doc: Floating-point maximum number across vector - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [fmaxnmv] - safety: - unsafe: [neon] - types: - - [float32x4_t, f32] - compose: - - LLVMLink: - name: "fmaxnmv.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fmaxnmv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vpmax{type[0]}" - doc: "Floating-point maximum pairwise" - arguments: ["a: {neon_type[1]}"] - return_type: "{type[2]}" - attr: [*neon-stable] - assert_instr: [fmaxp] - safety: - unsafe: [neon] - types: - - ["s_f32", float32x2_t, f32] - - ["qd_f64", float64x2_t, f64] - compose: - - LLVMLink: - name: "fmaxv.{type[0]}" - links: - - link: "llvm.aarch64.neon.fmaxv.{type[2]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vmin{neon_type.no}" - doc: "Minimum (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fmin] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fmin.{neon_type}" - links: - - link: "llvm.aarch64.neon.fmin.{neon_type}" - arch: aarch64,arm64ec - - - name: "vminnm{neon_type.no}" - doc: "Floating-point Minimum Number (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: [*neon-stable] - assert_instr: [fminnm] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "fminnm.{neon_type}" - links: - - link: "llvm.aarch64.neon.fminnm.{neon_type}" - arch: aarch64,arm64ec - - - name: "vminnmv{neon_type[0].no}" - doc: "Floating-point minimum number across vector" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, "f32"] - - [float64x2_t, "f64"] - compose: - - LLVMLink: - name: "vminnmv.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fminnmv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vminnmv{neon_type[0].no}" - doc: "Floating-point minimum number across vector" - arguments: ["a: {neon_type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmv]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x4_t, "f32"] - compose: - - LLVMLink: - name: "vminnmv.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fminnmv.{type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vmovl_high{neon_type[0].noq}" - doc: Vector move - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [sxtl2] - safety: - unsafe: [neon] - types: - - [int8x16_t, int16x8_t, int8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]'] - - [int32x4_t, int64x2_t, int32x2_t, '[2, 3]'] - compose: - - Let: - - a - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] - - FnCall: ["vmovl{neon_type[0].noq}", [a]] - - - name: "vmovl_high{neon_type[0].noq}" - doc: Vector move - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: [*neon-stable] - assert_instr: [uxtl2] - safety: - unsafe: [neon] - types: - - [uint8x16_t, uint16x8_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x8_t, uint32x4_t, uint16x4_t, '[4, 5, 6, 7]'] - - [uint32x4_t, uint64x2_t, uint32x2_t, '[2, 3]'] - compose: - - Let: - - a - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] - - FnCall: ["vmovl{neon_type[0].noq}", [a]] - - - name: "vpadd{neon_type.no}" - doc: Floating-point add pairwise - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{type}" - attr: [*neon-stable] - assert_instr: [faddp] - safety: - unsafe: [neon] - types: - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "faddp.{neon_type}" - links: - - link: "llvm.aarch64.neon.faddp.{neon_type}" - arch: aarch64,arm64ec - - - name: "vpadd{type[0]}" - doc: "Floating-point add pairwise" - arguments: ["a: {neon_type[1]}"] - return_type: "{type[2]}" - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["s_f32", float32x2_t, f32] - - ["d_f64", float64x2_t, f64] - compose: - - Let: - - a1 - - "{type[2]}" - - FnCall: [simd_extract!, [a, '0']] - - Let: - - a2 - - "{type[2]}" - - FnCall: [simd_extract!, [a, '1']] - - Identifier: ['a1 + a2', Symbol] - - - name: "vpmin{type[0]}" - doc: Floating-point minimum pairwise - arguments: ["a: {neon_type[1]}"] - return_type: "{type[2]}" - attr: [*neon-stable] - assert_instr: [fminp] - safety: - unsafe: [neon] - types: - - ["s_f32", float32x2_t, f32] - - ["qd_f64", float64x2_t, f64] - compose: - - LLVMLink: - name: "fminv.{type[2]}.{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.fminv.{type[2]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqdmullh_s16" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i16", "i32"] - compose: - - Let: [a, int16x4_t, {FnCall: [vdup_n_s16, [a]]}] - - Let: [b, int16x4_t, {FnCall: [vdup_n_s16, [b]]}] - - FnCall: [simd_extract!, [{FnCall: [vqdmull_s16, [a, b]]}, '0']] - - - name: "vqdmulls_s32" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i32", "i64"] - compose: - - LLVMLink: - name: "vqdmulls_s32" - links: - - link: "llvm.aarch64.neon.sqdmulls.scalar" - arch: aarch64,arm64ec - - - name: "vqdmull_high{neon_type[0].noq}" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]'] - - [int32x4_t, int64x2_t, int32x2_t, '[2, 3]'] - compose: - - Let: [a, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, '{type[3]}']]}] - - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, '{type[3]}']]}] - - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] - - - name: "vqdmull_high_n_{type[1]}" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int16x8_t, "i16", int32x4_t, int16x4_t, '[4, 5, 6, 7]'] - - [int32x4_t, "i32", int64x2_t, int32x2_t, '[2, 3]'] - compose: - - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - Let: [b, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[0].noq}", [b]]}] - - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] - - - name: "vqdmull{type[3]}" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, "i32", 'h_lane_s16', 'h_s16'] - - ["i32", int32x4_t, "i64", 's_laneq_s32', 's_s32'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 2]] - - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] - - FnCall: ["vqdmull{type[4]}", [a, b]] - - - name: "vqdmullh_laneq_s16" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, N = 4]]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x8_t, "i32"] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 3]] - - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] - - FnCall: ["vqdmullh_s16", [a, b]] - - - name: "vqdmulls_lane_s32" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i32", int32x2_t, "i64"] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 1]] - - Let: [b, "{type[0]}", {FnCall: [simd_extract!, [b, 'N as u32']]}] - - FnCall: ["vqdmulls_s32", [a, b]] - - - name: "vqdmull{type[6]}" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int16x4_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]', '[N as u32, N as u32, N as u32, N as u32]', '_high_lane_s16'] - - [int32x4_t, int32x4_t, int64x2_t, int32x2_t, '[2, 3]', '[N as u32, N as u32]', '_high_laneq_s32'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] - - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] - - - name: "vqdmull_high_lane_s32" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int32x2_t, int64x2_t, int32x2_t, '[2, 3]', '[N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '1']] - - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] - - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] - - - name: "vqdmull_high_laneq_s16" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull2, N = 4]]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int16x8_t, int32x4_t, int16x4_t, '[4, 5, 6, 7]', '[N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '3']] - - Let: [a, "{neon_type[3]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - Let: [b, "{neon_type[3]}", {FnCall: [simd_shuffle!, [b, b, "{type[5]}"]]}] - - FnCall: ["vqdmull{neon_type[0].noq}", [a, b]] - - - name: "vqdmull_laneq_s16" - doc: "Vector saturating doubling long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 4']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x8_t, int32x4_t, '[N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '3']] - - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - FnCall: [vqdmull_s16, [a, b]] - - - name: "vqdmull_laneq_s32" - doc: "Vector saturating doubling long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, int32x4_t, int64x2_t, '[N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - FnCall: [vqdmull_s32, [a, b]] - - - name: "vqdmlal{type[4]}" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_s16] - - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_s32] - - [int32x4_t, int16x8_t, "i16", int32x4_t, _high_n_s16] - - [int64x2_t, int32x4_t, "i32", int64x2_t, _high_n_s32] - compose: - - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] - - - name: "vqdmlal{type[4]}" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal2, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x4_t, int32x4_t, _high_lane_s16, '2'] - - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_laneq_s16, '3'] - - [int64x2_t, int32x4_t, int32x2_t, int64x2_t, _high_lane_s32, '1'] - - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_laneq_s32, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[5]}"]] - - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}::", [b, c]]}]] - - - name: "vqdmlalh_{type[2]}" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i32", "i16", "s16"] - compose: - - Let: [x, int32x4_t, {FnCall: [vqdmull_s16, [{FnCall: [vdup_n_s16, [b]]}, {FnCall: [vdup_n_s16, [c]]}]]}] - - FnCall: [vqadds_s32, [a, {FnCall: [simd_extract!, [x, 0]]}]] - - - name: "vqdmlals_s32" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "i32", "i32", "i64"] - compose: - - Let: [x, i64, {FnCall: [vqaddd_s64, [a, {FnCall: [vqdmulls_s32, [b, c]]}]]}] - - Identifier: ['x as i64', Symbol] - - - name: "vqdmlal{type[4]}" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {neon_type[2]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i32", "i16", int16x4_t, "i32", h_lane_s16, '2', h_s16] - - ["i32", "i16", int16x8_t, "i32", h_laneq_s16, '3', h_s16] - - ["i64", "i32", int32x2_t, "i64", s_lane_s32, '1', s_s32] - - ["i64", "i32", int32x4_t, "i64", s_laneq_s32, '2', s_s32] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[5]}"]] - - FnCall: ["vqdmlal{type[6]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vqdmlal_laneq_s16" - doc: "Vector widening saturating doubling multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x8_t, int32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '3']] - - FnCall: [vqaddq_s32, [a, {FnCall: ["vqdmull_laneq_s16::", [b, c]]}]] - - - name: "vqdmlal_laneq_s32" - doc: "Vector widening saturating doubling multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlal, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int32x2_t, int32x4_t, int64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - FnCall: [vqaddq_s64, [a, {FnCall: ["vqdmull_laneq_s32::", [b, c]]}]] - - - name: "vqdmlsl{type[4]}" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, _high_s16] - - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, _high_s32] - - [int32x4_t, int16x8_t, "i16", int32x4_t, _high_n_s16] - - [int64x2_t, int32x4_t, "i32", int64x2_t, _high_n_s32] - compose: - - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] - - - name: "vqdmlsl{type[4]}" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl2, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x4_t, int32x4_t, '_high_lane_s16', '2'] - - [int32x4_t, int16x8_t, int16x8_t, int32x4_t, '_high_laneq_s16', '3'] - - [int64x2_t, int32x4_t, int32x2_t, int64x2_t, '_high_lane_s32', '1'] - - [int64x2_t, int32x4_t, int32x4_t, int64x2_t, '_high_laneq_s32', '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[5]}"]] - - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}::", [b, c]]}]] - - - name: "vqdmlslh_s16" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i32", "i16"] - compose: - - Let: [x, int32x4_t, {FnCall: [vqdmull_s16, [{FnCall: [vdup_n_s16, [b]]}, {FnCall: [vdup_n_s16, [c]]}]]}] - - FnCall: [vqsubs_s32, [a, {FnCall: [simd_extract!, [x, '0']]}]] - - - name: "vqdmlsls_s32" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "i32", "i32", "i64"] - compose: - - Let: [x, i64, {FnCall: [vqsubd_s64, [a, {FnCall: [vqdmulls_s32, [b, c]]}]]}] - - Identifier: ['x as i64', Symbol] - - - name: "vqdmlsl{type[4]}" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {type[0]}", "b: {type[1]}", "c: {neon_type[2]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i32", "i16", int16x4_t, "i32", 'h_lane_s16', '2', 'h_s16'] - - ["i32", "i16", int16x8_t, "i32", 'h_laneq_s16', '3', 'h_s16'] - - ["i64", "i32", int32x2_t, "i64", 's_lane_s32', '1', 's_s32'] - - ["i64", "i32", int32x4_t, "i64", 's_laneq_s32', '2', 's_s32'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[5]}"]] - - FnCall: ["vqdmlsl{type[6]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vqdmlsl_laneq_s16" - doc: "Vector widening saturating doubling multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x8_t, int32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '3']] - - FnCall: ["vqsubq_s32", [a, {FnCall: ["vqdmull_laneq_s16::", [b, c]]}]] - - - name: "vqdmlsl_laneq_s32" - doc: "Vector widening saturating doubling multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmlsl, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int32x2_t, int32x4_t, int64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - FnCall: [vqsubq_s64, [a, {FnCall: ["vqdmull_laneq_s32::", [b, c]]}]] - - - name: "vqdmulh{type[4]}" - doc: "Signed saturating doubling multiply returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i16", "i16", "i16", int16x4_t, 'h_s16'] - - ["i32", "i32", "i32", int32x2_t, 's_s32'] - compose: - - Let: [a, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[3].no}", [a]]}] - - Let: [b, "{neon_type[3]}", {FnCall: ["vdup_n{neon_type[3].no}", [b]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqdmulh{neon_type[3].no}", [a, b]]}, '0']] - - - name: "vqdmulhh{type[3]}" - doc: "Signed saturating doubling multiply returning high half" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, "i16", '_lane_s16', '2'] - - ["i16", int16x8_t, "i16", '_laneq_s16', '3'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[4]}"]] - - Let: [b, 'i16', {FnCall: [simd_extract!, [b, 'N as u32']]}] - - FnCall: ['vqdmulhh_s16', [a, b]] - - - name: "vqdmulhs{type[3]}" - doc: "Signed saturating doubling multiply returning high half" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i32", int32x2_t, "i32", "_lane_s32", '1'] - - ["i32", int32x4_t, "i32", "_laneq_s32", '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[4]}"]] - - Let: [b, 'i32', {FnCall: [simd_extract!, [b, 'N as u32']]}] - - FnCall: ['vqdmulhs_s32', [a, b]] - - - name: "vqmovn_high{neon_type[1].noq}" - doc: "Signed saturating extract narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] - compose: - - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovn{neon_type[1].noq}", [b]]}, "{type[3]}"]] - - - name: "vqmovn_high{neon_type[1].noq}" - doc: "Signed saturating extract narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] - compose: - - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovn{neon_type[1].noq}", [b]]}, "{type[3]}"]] - - - name: "vqmovn{type[2]}" - doc: "Saturating extract narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i16", "i8", 'h_s16', s16] - - ["i32", "i16", 's_s32', s32] - compose: - - FnCall: [simd_extract!, [{FnCall: ["vqmovn_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] - - - name: "vqmovn{type[2]}" - doc: "Saturating extract narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u16", "u8", 'h_u16', 'u16'] - - ["u32", "u16", 's_u32', 'u32'] - compose: - - FnCall: [simd_extract!, [{FnCall: ["vqmovn_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] - - - name: "vqmovnd_s64" - doc: "Saturating extract narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "i32"] - compose: - - LLVMLink: - name: "vqmovnd_s64" - links: - - link: "llvm.aarch64.neon.scalar.sqxtn.i32.i64" - arch: aarch64,arm64ec - - - name: "vqmovnd_u64" - doc: "Saturating extract narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqxtn]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u64", "u32"] - compose: - - LLVMLink: - name: "vqmovnd_u64" - links: - - link: "llvm.aarch64.neon.scalar.uqxtn.i32.i64" - arch: aarch64,arm64ec - - - name: "vqmovun{type[2]}" - doc: "Signed saturating extract unsigned narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtun]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i16", "u8", 'h_s16', s16] - - ["i32", "u16", 's_s32', s32] - - ["i64", "u32", 'd_s64', s64] - compose: - - FnCall: [simd_extract!, [{FnCall: ["vqmovun_{type[3]}", [{FnCall: ["vdupq_n_{type[3]}", [a]]}]]}, '0']] - - - name: "vqmovun_high_{neon_type[1]}" - doc: "Signed saturating extract unsigned narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqxtun2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int16x8_t, uint8x16_t, s16, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, int32x4_t, uint16x8_t, s32, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, int64x2_t, uint32x4_t, s64, '[0, 1, 2, 3]'] - compose: - - FnCall: [simd_shuffle!, [a, {FnCall: ["vqmovun_{type[3]}", [b]]}, "{type[4]}"]] - - - name: "vqrdmulh{type[1]}" - doc: "Signed saturating rounding doubling multiply returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmulh]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i16", 'h_s16', 's16'] - - ["i32", 's_s32', 's32'] - compose: - - FnCall: [simd_extract!, [{FnCall: ["vqrdmulh_{type[2]}", [{FnCall: ["vdup_n_{type[2]}", [a]]}, {FnCall: ["vdup_n_{type[2]}", [b]]}]]}, '0']] - - - name: "vqrdmulh{type[2]}" - doc: "Signed saturating rounding doubling multiply returning high half" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmulh, LANE = 1]]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, 'h_lane_s16', 'h_s16', '2'] - - ["i16", int16x8_t, 'h_laneq_s16', 'h_s16', '3'] - - ["i32", int32x2_t, 's_lane_s32', 's_s32', '1'] - - ["i32", int32x4_t, 's_laneq_s32', 's_s32', '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] - - FnCall: ["vqrdmulh{type[3]}", [a, {FnCall: [simd_extract!, [b, 'LANE as u32']]}]] - - - name: "vqrdmlah{neon_type.no}" - doc: "Signed saturating rounding doubling multiply accumulate returning high half" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah]]}]] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - safety: - unsafe: [neon] - types: - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "vqrdmlah{neon_type.no}" - links: - - link: "llvm.aarch64.neon.sqrdmlah.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqrdmlah{type[3]}" - doc: "Signed saturating rounding doubling multiply accumulate returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah]]}]] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, s16, 'h_s16'] - - ["i32", int32x2_t, s32, 's_s32'] - compose: - - Let: [a, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [a]]}] - - Let: [b, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [b]]}] - - Let: [c, "{neon_type[1]}", {FnCall: ["vdup_n_{type[2]}", [c]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrdmlah_{type[2]}", [a, b, c]]}, '0']] - - - name: "vqrdmlah{type[0]}" - doc: "Signed saturating rounding doubling multiply accumulate returning high half" - arguments: ["a: {type[1]}", "b: {type[2]}", "c: {neon_type[3]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_s16, int16x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_s16, int16x8_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s16, int16x8_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_s32, int32x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_s32, int32x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_s32, int32x4_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s32, int32x4_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - Let: [c, "{type[1]}", {FnCall: [simd_shuffle!, [c, c, "{type[5]}"]]}] - - FnCall: ["vqrdmlah{neon_type[2].no}", [a, b, c]] - - - name: "vqrdmlah{type[4]}" - doc: "Signed saturating rounding doubling multiply accumulate returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlah, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, '2', "h_s16", h_lane_s16, h_s16] - - ["i16", int16x8_t, '3', "h_s16", h_laneq_s16, h_s16] - - ["i32", int32x2_t, '1', "s_s32", s_lane_s32, s_s32] - - ["i32", int32x4_t, '2', "s_s32", s_laneq_s32, s_s32] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - FnCall: ["vqrdmlah{type[5]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vqrdmlsh{neon_type.no}" - doc: "Signed saturating rounding doubling multiply subtract returning high half" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh]]}]] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - safety: - unsafe: [neon] - types: - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "vqrdmlsh{neon_type.no}" - links: - - link: "llvm.aarch64.neon.sqrdmlsh.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqrdmlsh{type[1]}" - doc: "Signed saturating rounding doubling multiply subtract returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh]]}]] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - safety: - unsafe: [neon] - types: - - ["i16", "h_s16", int16x4_t, s16] - - ["i32", "s_s32", int32x2_t, s32] - compose: - - Let: [a, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [a]]}] - - Let: [b, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [b]]}] - - Let: [c, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [c]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrdmlsh_{type[3]}", [a, b, c]]}, '0']] - - - name: "vqrdmlsh{type[0]}" - doc: "Signed saturating rounding doubling multiply subtract returning high half" - arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}", "c: {neon_type[3]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh, LANE = 1]]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_s16, int16x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_s16, int16x8_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s16, int16x8_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_s32, int32x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_s32, int32x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_s32, int32x4_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s32, int32x4_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - Let: [c, "{type[1]}", {FnCall: [simd_shuffle!, [c, c, "{type[5]}"]]}] - - FnCall: ["vqrdmlsh{neon_type[2].no}", [a, b, c]] - - - name: "vqrdmlsh{type[3]}" - doc: "Signed saturating rounding doubling multiply subtract returning high half" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [target_feature, ['enable = "rdm"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrdmlsh, LANE = 1]]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "rdm_intrinsics"', 'since = "1.62.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i16", int16x4_t, '2', h_lane_s16, h_s16] - - ["i16", int16x8_t, '3', h_laneq_s16, h_s16] - - ["i32", int32x2_t, '1', s_lane_s32, s_s32] - - ["i32", int32x4_t, '2', s_laneq_s32, s_s32] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - FnCall: ["vqrdmlsh{type[4]}", [a, b, {FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vqrshl{type[0]}" - doc: "Signed saturating rounding shift left" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['s_s32', "i32"] - - ['d_s64', "i64"] - compose: - - LLVMLink: - name: "vqrshl{type[0]}" - links: - - link: "llvm.aarch64.neon.sqrshl.{type[1]}" - arch: aarch64,arm64ec - - - name: "vqrshl{type[1]}" - doc: "Signed saturating rounding shift left" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i8", 'b_s8', int8x8_t, s8] - - ["i16", 'h_s16', int16x4_t, s16] - compose: - - Let: [a, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [a]]}] - - Let: [b, "{neon_type[2]}", {FnCall: ["vdup_n_{type[3]}", [b]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrshl_{type[3]}", [a, b]]}, '0']] - - - name: "vqrshl{type[2]}" - doc: "Unsigned signed saturating rounding shift left" - arguments: ["a: {type[0]}", "b: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u32", "i32", 's_u32'] - - ["u64", "i64", 'd_u64'] - compose: - - LLVMLink: - name: "vqrshl{type[2]}" - links: - - link: "llvm.aarch64.neon.uqrshl.{type[1]}" - arch: aarch64,arm64ec - - - name: "vqrshl{type[2]}" - doc: "Unsigned signed saturating rounding shift left" - arguments: ["a: {type[0]}", "b: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u8", "i8", "b_u8", uint8x8_t, int8x8_t, s8] - - ["u16", "i16", "h_u16", uint16x4_t, int16x4_t, s16] - compose: - - Let: [a, "{neon_type[3]}", {FnCall: ["vdup_n_{type[0]}", [a]]}] - - Let: [b, "{neon_type[4]}", {FnCall: ["vdup_n_{type[5]}", [b]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrshl_{type[0]}", [a, b]]}, '0']] - - - name: "vqrshrn{type[2]}" - doc: "Signed saturating rounded shift right narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i16", "i8", 'h_n_s16', 'N >= 1 && N <= 8', int16x8_t, q_n_s16] - - ["i32", "i16", 's_n_s32', 'N >= 1 && N <= 16', int32x4_t, q_n_s32] - - ["i64", "i32", 'd_n_s64', 'N >= 1 && N <= 32', int64x2_t, q_n_s64] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - Let: [a, "{neon_type[4]}", {FnCall: ["vdup{type[5]}", [a]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrshrn_n{neon_type[4].noq}::", [a]]}, '0']] - - - name: "vqrshrn{type[3]}" - doc: "Signed saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, '_high_n_s16', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]', 'N >= 1 && N <= 8'] - - [int16x4_t, int32x4_t, int16x8_t, '_high_n_s32', '[0, 1, 2, 3, 4, 5, 6, 7]', 'N >= 1 && N <= 16'] - - [int32x2_t, int64x2_t, int32x4_t, '_high_n_s64', '[0, 1, 2, 3]', 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[5]}"]] - - FnCall: [simd_shuffle!, [a, {FnCall: ["vqrshrn_n{neon_type[1].noq}::", [b]]}, "{type[4]}"]] - - - name: "vqrshrn{type[0]}" - doc: "Unsigned saturating rounded shift right narrow" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [h_n_u16, u16, u8, 'N >= 1 && N <= 8', uint16x8_t, q_n_u16, _n_u16] - - [s_n_u32, u32, u16, 'N >= 1 && N <= 16', uint32x4_t, q_n_u32, _n_u32] - - [d_n_u64, u64, u32, 'N >= 1 && N <= 32', uint64x2_t, q_n_u64, _n_u64] - compose: - - FnCall: [static_assert!, ['{type[3]}']] - - Let: [a, "{neon_type[4]}", {FnCall: ["vdup{type[5]}", [a]]}] - - FnCall: [simd_extract!, [{FnCall: ["vqrshrn{type[6]}::", [a]]}, '0']] - - - name: "vqrshrn_high_n{neon_type[1].noq}" - doc: "Unsigned saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ['{type[3]}']] - - FnCall: - - simd_shuffle! - - - a - - FnCall: - - "vqrshrn_n{neon_type[1].noq}::" - - - b - - "{type[4]}" - - - name: "vqrshrun{type[0]}" - doc: "Signed saturating rounded shift right unsigned narrow" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [h_n_s16, "i16", "u8", 'N >= 1 && N <= 8', int16x8_t, s16] - - [s_n_s32, "i32", "u16", 'N >= 1 && N <= 16', int32x4_t, s32] - - [d_n_s64, "i64", "u32", 'N >= 1 && N <= 32', int64x2_t, s64] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - Let: - - a - - "{neon_type[4]}" - - FnCall: ["vdupq_n_{type[5]}", [a]] - - FnCall: - - simd_extract! - - - FnCall: - - "vqrshrun_n_{type[5]}::" - - - a - - '0' - - - name: "vqrshrun_high_n{neon_type[1].noq}" - doc: "Signed saturating rounded shift right unsigned narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int16x8_t, uint8x16_t, 'N >= 1 && N <= 8', s16, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, int32x4_t, uint16x8_t, 'N >= 1 && N <= 16', s32, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, int64x2_t, uint32x4_t, 'N >= 1 && N <= 32', s64, '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: - - "vqrshrun_n_{type[4]}::" - - - b - - "{type[5]}" - - - name: "vqshld_{type}" - doc: "Signed saturating shift left" - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - i64 - compose: - - LLVMLink: - name: "vqshld{type}" - links: - - link: "llvm.aarch64.neon.sqshl.{type}" - arch: aarch64,arm64ec - - - name: "vqshl{type[0]}" - doc: "Signed saturating shift left" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [b_s8, "i8", int8x8_t] - - [h_s16, "i16", int16x4_t] - - [s_s32, "i32", int32x2_t] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: - - "vqshl{neon_type[2].noq}" - - - FnCall: ["vdup_n{neon_type[2].no}", [a]] - - FnCall: ["vdup_n{neon_type[2].no}", [b]] - - FnCall: [simd_extract!, [c, '0']] - - - name: "vqshl{type[0]}" - doc: "Signed saturating shift left" - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [b_n_s8, "i8", "3", s8] - - [h_n_s16, "i16", "4", s16] - - [s_n_s32, "i32", "5", s32] - - [d_n_s64, "i64", "6", s64] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] - - FnCall: - - simd_extract! - - - FnCall: - - "vqshl_n_{type[3]}::" - - - FnCall: ["vdup_n_{type[3]}", [a]] - - '0' - - - name: "vqshld_{type[0]}" - doc: "Unsigned saturating shift left" - arguments: ["a: {type[0]}", "b: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u64", "i64"] - compose: - - LLVMLink: - name: "vqshld{type[0]}" - links: - - link: "llvm.aarch64.neon.uqshl.{type[1]}" - arch: aarch64,arm64ec - - - name: "vqshl{type[0]}" - doc: "Unsigned saturating shift left" - arguments: ["a: {type[1]}", "b: {type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [b_u8, "u8", "i8", uint8x8_t, int8x8_t] - - [h_u16, "u16", "i16", uint16x4_t, int16x4_t] - - [s_u32, "u32", "i32", uint32x2_t, int32x2_t] - compose: - - Let: - - c - - "{neon_type[3]}" - - FnCall: - - "vqshl{neon_type[3].noq}" - - - FnCall: ["vdup{neon_type[3].N}", [a]] - - FnCall: ["vdup{neon_type[4].N}", [b]] - - FnCall: [simd_extract!, [c, '0']] - - - name: "vqshl{type[0]}" - doc: "Unsigned saturating shift left" - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [b_n_u8, "u8", '3'] - - [h_n_u16, "u16", '4'] - - [s_n_u32, "u32", '5'] - - [d_n_u64, "u64", '6'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] - - FnCall: - - simd_extract! - - - FnCall: ["vqshl_n_{type[1]}::", [{FnCall: ["vdup_n_{type[1]}", [a]]}]] - - '0' - - - name: "vqshrnd_n_s64" - doc: "Signed saturating shift right narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i64", "i32"] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vqshrnd{type[1]}" - arguments: - - "a: {type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.sqshrn.{type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqshrnd_n_s64", [a, N]] - - - name: "vqshrn{type[0]}" - doc: "Signed saturating shift right narrow" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [h_n_s16, "i16", "i8", 'N >= 1 && N <= 8', s16] - - [s_n_s32, "i32", "i16", 'N >= 1 && N <= 16', s32] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_extract! - - - FnCall: - - "vqshrn_n_{type[4]}::" - - - FnCall: ["vdupq_n_{type[4]}", [a]] - - '0' - - - name: "vqshrn{type[0]}" - doc: "Signed saturating shift right narrow" - arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] - return_type: "{neon_type[3]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_high_n_s16, int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]', s16] - - [_high_n_s32, int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]', s32] - - [_high_n_s64, int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]', s64] - compose: - - FnCall: [static_assert!, ["{type[4]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: ["vqshrn_n_{type[6]}::", [b]] - - "{type[5]}" - - - name: "vqshrnd_n_u64" - doc: "Unsigned saturating shift right narrow" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["u64", "u32"] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vqshrnd_n_u64" - arguments: - - "a: u64" - - "n: i32" - links: - - link: "llvm.aarch64.neon.uqshrn.i32" - arch: aarch64,arm64ec - - FnCall: ["_vqshrnd_n_u64", ["a.as_signed()", N]] - - - name: "vqshrn{type[0]}" - doc: "Unsigned saturating shift right narrow" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ['h_n_u16', "u16", "u8", 'N >= 1 && N <= 8'] - - ['s_n_u32', "u32", "u16", 'N >= 1 && N <= 16'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - "simd_extract!" - - - FnCall: - - "vqshrn_n_{type[1]}::" - - - FnCall: ["vdupq_n_{type[1]}", [a]] - - '0' - - - name: "vqshrn{type[0]}" - doc: "Unsigned saturating shift right narrow" - arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] - return_type: "{neon_type[3]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_high_n_u16, uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [_high_n_u32, uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [_high_n_u64, uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[4]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: ["vqshrn_n_{neon_type[2]}::", [b]] - - "{type[5]}" - - - name: "vqshrun{type[0]}" - doc: "Signed saturating shift right unsigned narrow" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [h_n_s16, "i16", "u8", 'N >= 1 && N <= 8', s16] - - [s_n_s32, "i32", "u16", 'N >= 1 && N <= 16', s32] - - [d_n_s64, "i64", "u32", 'N >= 1 && N <= 32', s64] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_extract! - - - FnCall: - - "vqshrun_n_{type[4]}::" - - - FnCall: ["vdupq_n_{type[4]}", [a]] - - '0' - - - name: "vqshrun_high_n_{neon_type[1]}" - doc: "Signed saturating shift right unsigned narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, int32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, int64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: ["vqshrun_n_{neon_type[1]}::", [b]] - - "{type[4]}" - - - name: "vsqadd{type[0]}" - doc: "Unsigned saturating accumulate of signed value" - arguments: ["a: {type[1]}", "b: {type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usqadd]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [b_u8, "u8", "i8", s8] - - [h_u16, "u16", "i16", s16] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vsqadd_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[2]}", [b]] - - '0' - - - name: "vsqadd{type[0]}" - doc: "Unsigned saturating accumulate of signed value" - arguments: ["a: {type[1]}", "b: {type[2]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [usqadd]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [s_u32, "u32", "i32"] - - [d_u64, "u64", "i64"] - compose: - - LLVMLink: - name: "vsqadd{type[0]}" - links: - - link: "llvm.aarch64.neon.usqadd.{type[2]}" - arch: aarch64,arm64ec - - - name: "vsqrt{neon_type.no}" - doc: "Calculates the square root of each lane." - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fsqrt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_fsqrt, [a]] - - - name: "vrsqrts{type[0]}" - doc: "Floating-point reciprocal square root step" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrts]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [_f64, float64x1_t, v1f64] - - [q_f64, float64x2_t, v2f64] - compose: - - LLVMLink: - name: "vrsqrts{type[0]}" - links: - - link: "llvm.aarch64.neon.frsqrts.{type[2]}" - arch: aarch64,arm64ec - - - name: "vrsqrts{type[0]}" - doc: "Floating-point reciprocal square root step" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrts]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [s_f32, "f32"] - - [d_f64, "f64"] - compose: - - LLVMLink: - name: "vrsqrts{type[0]}" - links: - - link: "llvm.aarch64.neon.frsqrts.{type[1]}" - arch: aarch64,arm64ec - - - name: "vrecpe{type[0]}" - doc: "Reciprocal estimate." - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpe]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [_f64, float64x1_t, v1f64] - - [q_f64, float64x2_t, v2f64] - compose: - - LLVMLink: - name: "vrecpe{type[0]}" - links: - - link: "llvm.aarch64.neon.frecpe.{type[2]}" - arch: aarch64,arm64ec - - - name: "vrecpe{type[0]}" - doc: "Reciprocal estimate." - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpe]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [s_f32, "f32"] - - [d_f64, "f64"] - compose: - - LLVMLink: - name: "vrecpe{type[0]}" - links: - - link: "llvm.aarch64.neon.frecpe.{type[1]}" - arch: aarch64,arm64ec - - - name: "vrecps{type[0]}" - doc: "Floating-point reciprocal step" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecps]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [_f64, float64x1_t, v1f64] - - [q_f64, float64x2_t, v2f64] - compose: - - LLVMLink: - name: "vrecps{type[0]}" - links: - - link: "llvm.aarch64.neon.frecps.{type[2]}" - arch: aarch64,arm64ec - - - name: "vrecps{type[0]}" - doc: "Floating-point reciprocal step" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecps]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [s_f32, "f32"] - - [d_f64, "f64"] - compose: - - LLVMLink: - name: "vrecps{type[0]}" - links: - - link: "llvm.aarch64.neon.frecps.{type[1]}" - arch: aarch64,arm64ec - - - name: "vrecpx{type[0]}" - doc: "Floating-point reciprocal exponent" - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frecpx]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [s_f32, "f32"] - - [d_f64, "f64"] - compose: - - LLVMLink: - name: "vrecpxs{type[0]}" - links: - - link: "llvm.aarch64.neon.frecpx.{type[1]}" - arch: aarch64,arm64ec - - - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" - doc: Vector reinterpret cast operation - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: [*neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [poly64x1_t, int64x1_t] - - [poly64x1_t, uint64x1_t] - - [int64x1_t, poly64x1_t] - - [uint64x1_t, poly64x1_t] - - [poly64x2_t, int64x2_t] - - [poly64x2_t, uint64x2_t] - - [int64x2_t, poly64x2_t] - - [uint64x2_t, poly64x2_t] - - [float64x1_t, int8x8_t] - - [float64x1_t, int16x4_t] - - [float64x1_t, int32x2_t] - - [float64x1_t, int64x1_t] - - [float64x2_t, int8x16_t] - - [float64x2_t, int16x8_t] - - [float64x2_t, int32x4_t] - - [float64x2_t, int64x2_t] - - [float64x1_t, uint8x8_t] - - [float64x1_t, uint16x4_t] - - [float64x1_t, uint32x2_t] - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint8x16_t] - - [float64x2_t, uint16x8_t] - - [float64x2_t, uint32x4_t] - - [float64x2_t, uint64x2_t] - - [float64x1_t, poly8x8_t] - - [float64x1_t, poly16x4_t] - - [float32x2_t, poly64x1_t] - - [float64x1_t, poly64x1_t] - - [float64x2_t, poly8x16_t] - - [float64x2_t, poly16x8_t] - - [float32x4_t, poly64x2_t] - - [float64x2_t, poly64x2_t] - - [float64x2_t, p128] - - [int8x8_t, float64x1_t] - - [int16x4_t, float64x1_t] - - [int32x2_t, float64x1_t] - - [int64x1_t, float64x1_t] - - [int8x16_t, float64x2_t] - - [int16x8_t, float64x2_t] - - [int32x4_t, float64x2_t] - - [int64x2_t, float64x2_t] - - [poly8x8_t, float64x1_t] - - [uint16x4_t, float64x1_t] - - [uint32x2_t, float64x1_t] - - [uint64x1_t, float64x1_t] - - [poly8x16_t, float64x2_t] - - [uint16x8_t, float64x2_t] - - [uint32x4_t, float64x2_t] - - [uint64x2_t, float64x2_t] - - [uint8x8_t, float64x1_t] - - [poly16x4_t, float64x1_t] - - [poly64x1_t, float64x1_t] - - [poly64x1_t, float32x2_t] - - [uint8x16_t, float64x2_t] - - [poly16x8_t, float64x2_t] - - [poly64x2_t, float64x2_t] - - [poly64x2_t, float32x4_t] - - [p128, float64x2_t] - - [float32x2_t, float64x1_t] - - [float64x1_t, float32x2_t] - - [float32x4_t, float64x2_t] - - [float64x2_t, float32x4_t] - compose: - - FnCall: [transmute, [a]] - - - name: "vrshld_s64" - doc: "Signed rounding shift left" - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - "i64" - compose: - - LLVMLink: - name: "vrshld_{type}" - links: - - link: "llvm.aarch64.neon.srshl.{type}" - arch: aarch64,arm64ec - - - name: "vrshld_{type[0]}" - doc: "Unsigned rounding shift left" - arguments: ["a: {type[0]}", "b: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u64", "i64"] - compose: - - LLVMLink: - name: "vrshld_{type[0]}" - links: - - link: "llvm.aarch64.neon.urshl.{type[1]}" - arch: aarch64,arm64ec - - - name: "vrshrd_n_s64" - doc: "Signed rounding shift right" - arguments: ["a: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - ["i64", 'N >= 1 && N <= 64', '-N as i64'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - FnCall: [vrshld_s64, [a, "{type[2]}"]] - - - name: "vrshrd_n_u64" - doc: "Unsigned rounding shift right" - arguments: ["a: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - "u64" - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - FnCall: ["vrshld_u64", [a, '-N as i64']] - - - name: "vrshrn_high_n_{neon_type[1]}" - doc: "Rounding shift right narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rshrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: ["vrshrn_n_{neon_type[1]}::", [b]] - - "{type[4]}" - - - name: "vrsubhn_high_{neon_type[1]}" - doc: "Rounding subtract returning high narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[3]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rsubhn2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] - - [uint8x8_t, uint16x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] - compose: - - Let: - - x - - "{neon_type[0]}" - - FnCall: ["vrsubhn_{neon_type[1]}", [b, c]] - - FnCall: [simd_shuffle!, [a, x, "{type[4]}"]] - - - name: "vcopy{neon_type[0].lane_nox}" - doc: "Insert vector element from another vector element" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 0', 'LANE2 = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1', '3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE1: i32, const LANE2: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, int8x8_t, int8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int16x4_t, int16x4_t, int16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int32x2_t, int32x2_t, int32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint8x8_t, uint8x8_t, uint8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint16x4_t, uint16x4_t, uint16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint32x2_t, uint32x2_t, uint32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly8x8_t, poly8x8_t, poly8x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly16x4_t, poly16x4_t, poly16x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [float32x2_t, float32x2_t, float32x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int8x16_t, int8x8_t, int8x16_t, '4', '3', ' let b: int8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int16x8_t, int16x4_t, int16x8_t, '3', '2', ' let b: int16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int32x4_t, int32x2_t, int32x4_t, '2', '1', ' let b: int32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint8x16_t, uint8x8_t, uint8x16_t, '4', '3', ' let b: uint8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint16x8_t, uint16x4_t, uint16x8_t, '3', '2', ' let b: uint16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint32x4_t, uint32x2_t, uint32x4_t, '2', '1', ' let b: uint32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly8x16_t, poly8x8_t, poly8x16_t, '4', '3', ' let b: poly8x16_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly16x8_t, poly16x4_t, poly16x8_t, '3', '2', ' let b: poly16x8_t = simd_shuffle!(b, b, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE1, '{type[3]}']] - - FnCall: [static_assert_uimm_bits!, [LANE2, '{type[4]}']] - - Identifier: ["{type[5]}", Symbol] - - - name: "vcopy{neon_type[0].laneq_nox}" - doc: "Insert vector element from another vector element" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 0', 'LANE2 = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1', '3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE1: i32, const LANE2: i32'] - safety: - unsafe: [neon] - types: - - [int8x16_t, int8x16_t, int8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int16x8_t, int16x8_t, int16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int32x4_t, int32x4_t, int32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int64x2_t, int64x2_t, int64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint8x16_t, uint8x16_t, uint8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint16x8_t, uint16x8_t, uint16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint32x4_t, uint32x4_t, uint32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint64x2_t, uint64x2_t, uint64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly8x16_t, poly8x16_t, poly8x16_t, '4', '4', ' match LANE1 & 0b1111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32, 8, 9, 10, 11, 12, 13, 14, 15]), 8 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 16 + LANE2 as u32, 9, 10, 11, 12, 13, 14, 15]), 9 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 16 + LANE2 as u32, 10, 11, 12, 13, 14, 15]), 10 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 16 + LANE2 as u32, 11, 12, 13, 14, 15]), 11 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 16 + LANE2 as u32, 12, 13, 14, 15]), 12 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16 + LANE2 as u32, 13, 14, 15]), 13 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16 + LANE2 as u32, 14, 15]), 14 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16 + LANE2 as u32, 15]), 15 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly16x8_t, poly16x8_t, poly16x8_t, '3', '3', ' match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 8 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 8 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 8 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly64x2_t, poly64x2_t, poly64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [float32x4_t, float32x4_t, float32x4_t, '2', '2', ' match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [float64x2_t, float64x2_t, float64x2_t, '1', '1', ' match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int8x8_t, int8x16_t, int8x8_t, '3', '4', ' let a: int8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int16x4_t, int16x8_t, int16x4_t, '2', '3', ' let a: int16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [int32x2_t, int32x4_t, int32x2_t, '1', '2', ' let a: int32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint8x8_t, uint8x16_t, uint8x8_t, '3', '4', ' let a: uint8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint16x4_t, uint16x8_t, uint16x4_t, '2', '3', ' let a: uint16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint32x2_t, uint32x4_t, uint32x2_t, '1', '2', 'let a: uint32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly8x8_t, poly8x16_t, poly8x8_t, '3', '4', ' let a: poly8x16_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); match LANE1 & 0b111 { 0 => simd_shuffle!(a, b, [16 + LANE2 as u32, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [0, 16 + LANE2 as u32, 2, 3, 4, 5, 6, 7]), 2 => simd_shuffle!(a, b, [0, 1, 16 + LANE2 as u32, 3, 4, 5, 6, 7]), 3 => simd_shuffle!(a, b, [0, 1, 2, 16 + LANE2 as u32, 4, 5, 6, 7]), 4 => simd_shuffle!(a, b, [0, 1, 2, 3, 16 + LANE2 as u32, 5, 6, 7]), 5 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 16 + LANE2 as u32, 6, 7]), 6 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 16 + LANE2 as u32, 7]), 7 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 16 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly16x4_t, poly16x8_t, poly16x4_t, '2', '3', ' let a: poly16x8_t = simd_shuffle!(a, a, [0, 1, 2, 3, 4, 5, 6, 7]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [8 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 8 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 8 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 8 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [float32x2_t, float32x4_t, float32x2_t, '1', '2', ' let a: float32x4_t = simd_shuffle!(a, a, [0, 1, 2, 3]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE1, '{type[3]}']] - - FnCall: [static_assert_uimm_bits!, [LANE2, '{type[4]}']] - - Identifier: ["{type[5]}", Symbol] - - - name: "vcopyq_lane_{neon_type[0]}" - doc: "Insert vector element from another vector element" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 1', 'LANE2 = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1', '3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE1: i32, const LANE2: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int64x1_t, 'let b: int64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [uint64x2_t, uint64x1_t, 'let b: uint64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [poly64x2_t, poly64x1_t, 'let b: poly64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - - [float64x2_t, float64x1_t, ' let b: float64x2_t = simd_shuffle!(b, b, [0, 1]); match LANE1 & 0b1 { 0 => simd_shuffle!(a, b, [2 + LANE2 as u32, 1]), 1 => simd_shuffle!(a, b, [0, 2 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE1, '1']] - - FnCall: [static_assert!, ['LANE2 == 0']] - - Identifier: ['{type[2]}', Symbol] - - - name: "vcopyq_lane_f32" - doc: "Insert vector element from another vector element" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [mov, 'LANE1 = 1', 'LANE2 = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1', '3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE1: i32, const LANE2: i32'] - safety: - unsafe: [neon] - types: - - [float32x4_t, float32x2_t, ' let b: float32x4_t = simd_shuffle!(b, b, [0, 1, 2, 3]); match LANE1 & 0b11 { 0 => simd_shuffle!(a, b, [4 + LANE2 as u32, 1, 2, 3]), 1 => simd_shuffle!(a, b, [0, 4 + LANE2 as u32, 2, 3]), 2 => simd_shuffle!(a, b, [0, 1, 4 + LANE2 as u32, 3]), 3 => simd_shuffle!(a, b, [0, 1, 2, 4 + LANE2 as u32]), _ => unreachable_unchecked(), }'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE1, 2]] - - FnCall: [static_assert_uimm_bits!, [LANE2, 1]] - - Identifier: ["{type[2]}", Symbol] - - - name: "vcreate_f64" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u64", float64x1_t] - compose: - - FnCall: [transmute, [a]] - - - name: "vset_lane_f64" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f64", float64x1_t, float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vsetq_lane_f64" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f64", float64x2_t, float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vshld_s64" - doc: "Signed Shift left" - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sshl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - "i64" - compose: - - FnCall: - - transmute - - - FnCall: - - vshl_s64 - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vshld_{type[0]}" - doc: "Unsigned Shift left" - arguments: ["a: {type[0]}", "b: {type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ushl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["u64", "i64"] - compose: - - FnCall: - - transmute - - - FnCall: - - vshl_u64 - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vshll_high_n_{neon_type[0]}" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sshll2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x16_t, int16x8_t, int8x8_t, 'N >= 0 && N <= 8', '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x8_t, int32x4_t, int16x4_t, 'N >= 0 && N <= 16', '[4, 5, 6, 7]'] - - [int32x4_t, int64x2_t, int32x2_t, 'N >= 0 && N <= 32', '[2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - FnCall: ["vshll_n_{neon_type[2]}::", [b]] - - - name: "vshll_high_n_{neon_type[0]}" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ushll2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x16_t, uint16x8_t, uint8x8_t, 'N >= 0 && N <= 8', '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x8_t, uint32x4_t, uint16x4_t, 'N >= 0 && N <= 16', '[4, 5, 6, 7]'] - - [uint32x4_t, uint64x2_t, uint32x2_t, 'N >= 0 && N <= 32', '[2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - Let: [b, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[4]}"]]}] - - FnCall: ["vshll_n_{neon_type[2]}::", [b]] - - - name: "vshrn_high_n_{neon_type[1]}" - doc: "Shift right narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [shrn2, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - - [uint8x8_t, uint16x8_t, uint8x16_t, 'N >= 1 && N <= 8', '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, 'N >= 1 && N <= 16', '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, 'N >= 1 && N <= 32', '[0, 1, 2, 3]'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_shuffle! - - - a - - FnCall: ["vshrn_n_{neon_type[1]}::", [b]] - - "{type[4]}" - - - name: "vsm3partw1{neon_type.no}" - doc: "SM3PARTW1" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sm4"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3partw1]]}]] - - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] - safety: - unsafe: [neon] - types: - - uint32x4_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sm3partw1 - links: - - link: "llvm.aarch64.crypto.sm3partw1" - arch: aarch64,arm64ec - - - name: "vsm3partw2{neon_type.no}" - doc: "SM3PARTW2" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sm4"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3partw2]]}]] - - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] - safety: - unsafe: [neon] - types: - - uint32x4_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sm3partw2 - links: - - link: "llvm.aarch64.crypto.sm3partw2" - arch: aarch64,arm64ec - - - name: "vsm3ss1{neon_type.no}" - doc: "SM3SS1" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sm4"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm3ss1]]}]] - - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] - safety: - unsafe: [neon] - types: - - uint32x4_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sm3ss1 - links: - - link: "llvm.aarch64.crypto.sm3ss1" - arch: aarch64,arm64ec - - - name: "vsm4ekey{neon_type.no}" - doc: "SM4 key" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sm4"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm4ekey]]}]] - - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] - safety: - unsafe: [neon] - types: - - uint32x4_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sm4ekey - links: - - link: "llvm.aarch64.crypto.sm4ekey" - arch: aarch64,arm64ec - - - name: "vsm4e{neon_type.no}" - doc: "SM4 encode" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sm4"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sm4e]]}]] - - FnCall: [unstable, ['feature = "stdarch_neon_sm4"', 'issue = "117226"']] - safety: - unsafe: [neon] - types: - - uint32x4_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sm4e - links: - - link: "llvm.aarch64.crypto.sm4e" - arch: aarch64,arm64ec - - - name: "vrax1{neon_type.no}" - doc: "Rotate and exclusive OR" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rax1]]}]] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - safety: - unsafe: [neon] - types: - - uint64x2_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.rax1 - links: - - link: "llvm.aarch64.crypto.rax1" - arch: aarch64,arm64ec - - - name: "vsha512h{neon_type.no}" - doc: "SHA512 hash update part 1" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512h]]}]] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - safety: - unsafe: [neon] - types: - - uint64x2_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sha512h - links: - - link: "llvm.aarch64.crypto.sha512h" - arch: aarch64,arm64ec - - - name: "vsha512h2{neon_type.no}" - doc: "SHA512 hash update part 2" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512h2]]}]] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - safety: - unsafe: [neon] - types: - - uint64x2_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sha512h2 - links: - - link: "llvm.aarch64.crypto.sha512h2" - arch: aarch64,arm64ec - - - name: "vsha512su0{neon_type.no}" - doc: "SHA512 schedule update 0" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512su0]]}]] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - safety: - unsafe: [neon] - types: - - uint64x2_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sha512su0 - links: - - link: "llvm.aarch64.crypto.sha512su0" - arch: aarch64,arm64ec - - - name: "vsha512su1{neon_type.no}" - doc: "SHA512 schedule update 1" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,sha3"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sha512su1]]}]] - - FnCall: [stable, ['feature = "stdarch_neon_sha3"', 'since = "1.79.0"']] - safety: - unsafe: [neon] - types: - - uint64x2_t - compose: - - LLVMLink: - name: llvm.aarch64.crypto.sha512su1 - links: - - link: "llvm.aarch64.crypto.sha512su1" - arch: aarch64,arm64ec - - - name: "vrnd32x{neon_type.no}" - doc: "Floating-point round to 32-bit integer, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32x]]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "vrnd32x{neon_type.no}" - links: - - link: "llvm.aarch64.neon.frint32x.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnd32x{neon_type.no}" - doc: "Floating-point round to 32-bit integer, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32x]]}]] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - LLVMLink: - name: "vrnd32x{neon_type.no}" - arguments: - - "a: f64" - return_type: "f64" - links: - - link: "llvm.aarch64.frint32x.f64" - arch: aarch64,arm64ec - - FnCall: - - transmute - - - FnCall: - - _vrnd32x_f64 - - - FnCall: [simd_extract!, [a, 0]] - - - name: "vrnd32z{neon_type.no}" - doc: "Floating-point round to 32-bit integer toward zero" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32z]]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "vrnd32z{neon_type.no}" - links: - - link: "llvm.aarch64.neon.frint32z.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnd32z{neon_type.no}" - doc: "Floating-point round to 32-bit integer toward zero" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint32z]]}]] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - LLVMLink: - name: "vrnd32z{neon_type.no}" - arguments: - - "a: f64" - return_type: "f64" - links: - - link: "llvm.aarch64.frint32z.f64" - arch: aarch64,arm64ec - - FnCall: - - transmute - - - FnCall: [_vrnd32z_f64, [{FnCall: [simd_extract!, [a, 0]]}]] - - - name: "vrnd64x{neon_type.no}" - doc: "Floating-point round to 64-bit integer, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64x]]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "vrnd64x{neon_type.no}" - links: - - link: "llvm.aarch64.neon.frint64x.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnd64x{neon_type.no}" - doc: "Floating-point round to 64-bit integer, using current rounding mode" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64x]]}]] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - LLVMLink: - name: "vrnd64x{neon_type.no}" - arguments: - - "a: f64" - return_type: "f64" - links: - - link: "llvm.aarch64.frint64x.f64" - arch: aarch64,arm64ec - - FnCall: - - transmute - - - FnCall: [_vrnd64x_f64, [{FnCall: [simd_extract!, [a, 0]]}]] - - - name: "vrnd64z{neon_type.no}" - doc: "Floating-point round to 64-bit integer toward zero" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64z]]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - - float64x2_t - compose: - - LLVMLink: - name: "vrnd64z{neon_type.no}" - links: - - link: "llvm.aarch64.neon.frint64z.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrnd64z{neon_type.no}" - doc: "Floating-point round to 64-bit integer toward zero" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [target_feature, ['enable = "neon,frintts"']] - - FnCall: [unstable, ['feature = "stdarch_neon_ftts"', 'issue = "117227"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [frint64z]]}]] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - LLVMLink: - name: "vrnd64z{neon_type.no}" - arguments: - - "a: f64" - return_type: "f64" - links: - - link: "llvm.aarch64.frint64z.f64" - arch: aarch64,arm64ec - - FnCall: - - transmute - - - FnCall: [_vrnd64z_f64, [{FnCall: [simd_extract!, [a, 0]]}]] - - - name: "vtrn1{neon_type[0].no}" - doc: Transpose vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [trn1]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [int8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] - - [int16x4_t, '[0, 4, 2, 6]'] - - [int16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [int32x4_t, '[0, 4, 2, 6]'] - - [uint8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [uint8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] - - [uint16x4_t, '[0, 4, 2, 6]'] - - [uint16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [uint32x4_t, '[0, 4, 2, 6]'] - - [poly8x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [poly8x16_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]'] - - [poly16x4_t, '[0, 4, 2, 6]'] - - [poly16x8_t, '[0, 8, 2, 10, 4, 12, 6, 14]'] - - [float32x4_t, '[0, 4, 2, 6]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vtrn1{neon_type[0].no}" - doc: Transpose vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, '[0, 2]'] - - [int64x2_t, '[0, 2]'] - - [uint32x2_t, '[0, 2]'] - - [uint64x2_t, '[0, 2]'] - - [poly64x2_t, '[0, 2]'] - - [float32x2_t, '[0, 2]'] - - [float64x2_t, '[0, 2]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vtrn2{neon_type[0].no}" - doc: Transpose vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [trn2]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [int8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [int16x4_t, '[1, 5, 3, 7]'] - - [int16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [int32x4_t, '[1, 5, 3, 7]'] - - [uint8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [uint8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [uint16x4_t, '[1, 5, 3, 7]'] - - [uint16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [uint32x4_t, '[1, 5, 3, 7]'] - - [poly8x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [poly8x16_t, '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [poly16x4_t, '[1, 5, 3, 7]'] - - [poly16x8_t, '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [float32x4_t, '[1, 5, 3, 7]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vtrn2{neon_type[0].no}" - doc: Transpose vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, '[1, 3]'] - - [int64x2_t, '[1, 3]'] - - [uint32x2_t, '[1, 3]'] - - [uint64x2_t, '[1, 3]'] - - [poly64x2_t, '[1, 3]'] - - [float32x2_t, '[1, 3]'] - - [float64x2_t, '[1, 3]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vzip2{neon_type[0].no}" - doc: Zip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [int8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [int16x4_t, '[2, 6, 3, 7]'] - - [int16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [int32x2_t, '[1, 3]'] - - [int32x4_t, '[2, 6, 3, 7]'] - - [int64x2_t, '[1, 3]'] - - [uint8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [uint8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [uint16x4_t, '[2, 6, 3, 7]'] - - [uint16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [uint32x2_t, '[1, 3]'] - - [uint32x4_t, '[2, 6, 3, 7]'] - - [uint64x2_t, '[1, 3]'] - - [poly8x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [poly8x16_t, '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [poly16x4_t, '[2, 6, 3, 7]'] - - [poly16x8_t, '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [poly64x2_t, '[1, 3]'] - - [float32x2_t, '[1, 3]'] - - [float32x4_t, '[2, 6, 3, 7]'] - - [float64x2_t, '[1, 3]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vzip1{neon_type[0].no}" - doc: Zip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [int8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] - - [int16x4_t, '[0, 4, 1, 5]'] - - [int16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [int32x2_t, '[0, 2]'] - - [int32x4_t, '[0, 4, 1, 5]'] - - [int64x2_t, '[0, 2]'] - - [uint8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [uint8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] - - [uint16x4_t, '[0, 4, 1, 5]'] - - [uint16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [uint32x2_t, '[0, 2]'] - - [uint32x4_t, '[0, 4, 1, 5]'] - - [uint64x2_t, '[0, 2]'] - - [poly8x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [poly8x16_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]'] - - [poly16x4_t, '[0, 4, 1, 5]'] - - [poly16x8_t, '[0, 8, 1, 9, 2, 10, 3, 11]'] - - [poly64x2_t, '[0, 2]'] - - [float32x2_t, '[0, 2]'] - - [float32x4_t, '[0, 4, 1, 5]'] - - [float64x2_t, '[0, 2]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vuzp1{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip1]]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, '[0, 2]'] - - [int64x2_t, '[0, 2]'] - - [uint32x2_t, '[0, 2]'] - - [uint64x2_t, '[0, 2]'] - - [poly64x2_t, '[0, 2]'] - - [float32x2_t, '[0, 2]'] - - [float64x2_t, '[0, 2]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vuzp1{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uzp1]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [int8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] - - [int16x4_t, '[0, 2, 4, 6]'] - - [int16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [int32x4_t, '[0, 2, 4, 6]'] - - [uint8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [uint8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] - - [uint16x4_t, '[0, 2, 4, 6]'] - - [uint16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [uint32x4_t, '[0, 2, 4, 6] '] - - [poly8x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [poly8x16_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]'] - - [poly16x4_t, '[0, 2, 4, 6]'] - - [poly16x8_t, '[0, 2, 4, 6, 8, 10, 12, 14]'] - - [float32x4_t, '[0, 2, 4, 6]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vuzp2{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [zip2]]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, '[1, 3]'] - - [int64x2_t, '[1, 3]'] - - [uint32x2_t, '[1, 3]'] - - [uint64x2_t, '[1, 3]'] - - [poly64x2_t, '[1, 3]'] - - [float32x2_t, '[1, 3]'] - - [float64x2_t, '[1, 3]'] - compose: - - FnCall: ["simd_shuffle!", [a, b, "{type[1]}"]] - - - name: "vuzp2{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uzp2]]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [int8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [int16x4_t, '[1, 3, 5, 7]'] - - [int16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [int32x4_t, '[1, 3, 5, 7]'] - - [uint8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [uint8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [uint16x4_t, '[1, 3, 5, 7]'] - - [uint16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [uint32x4_t, '[1, 3, 5, 7]'] - - [poly8x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [poly8x16_t, '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [poly16x4_t, '[1, 3, 5, 7]'] - - [poly16x8_t, '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [float32x4_t, '[1, 3, 5, 7]'] - compose: - - FnCall: - - "simd_shuffle!" - - - a - - b - - "{type[1]}" - - - name: "vabal_high_{neon_type[1]}" - doc: "Unsigned Absolute difference and Accumulate Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [uabal]]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x16_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]', '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint32x4_t, uint16x8_t, uint16x4_t, '[4, 5, 6, 7]', '[4, 5, 6, 7]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '[2, 3]', '[2, 3]'] - compose: - - Let: - - d - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [b, b, "{type[3]}"]] - - Let: - - e - - "{neon_type[2]}" - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - Let: [f, "{neon_type[2]}", {FnCall: ["vabd_{neon_type[2]}", [d, e]]}] - - FnCall: - - simd_add - - - a - - FnCall: [simd_cast, [f]] - - - name: "vabal_high{neon_type[1].noq}" - doc: Signed Absolute difference and Accumulate Long - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sabal]]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x16_t, int8x16_t, '[8, 9, 10, 11, 12, 13, 14, 15]', int8x8_t, uint8x8_t] - - [int32x4_t, int16x8_t, int16x8_t, '[4, 5, 6, 7]', int16x4_t, uint16x4_t] - - [int64x2_t, int32x4_t, int32x4_t, '[2, 3]', int32x2_t, uint32x2_t] - compose: - - Let: - - d - - "{neon_type[4]}" - - FnCall: - - simd_shuffle! - - - b - - b - - "{type[3]}" - - Let: - - e - - "{neon_type[4]}" - - FnCall: - - simd_shuffle! - - - c - - c - - "{type[3]}" - - Let: - - f - - "{neon_type[4]}" - - FnCall: - - "vabd{neon_type[4].no}" - - - d - - e - - Let: - - f - - "{neon_type[5]}" - - FnCall: - - simd_cast - - - f - - FnCall: - - simd_add - - - a - - FnCall: - - simd_cast - - - f - - - name: "vqabs{neon_type.no}" - doc: Signed saturating Absolute value - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-stable - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] - safety: - unsafe: [neon] - types: - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "sqabs.{neon_type}" - links: - - link: "llvm.aarch64.neon.sqabs.{neon_type}" - arch: aarch64,arm64ec - - - name: "vslid_n_{type}" - doc: Shift left and insert - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - static_defs: - - "const N: i32" - attr: - - *neon-stable - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sli, 'N = 2']]}]] - safety: - unsafe: [neon] - types: - - i64 - - u64 - compose: - - FnCall: - - "static_assert!" - - - 'N >= 0 && N <= 63' - - FnCall: - - transmute - - - FnCall: - - "vsli_n_{type}::" - - - FnCall: - - transmute - - - a - - FnCall: - - transmute - - - b - - - name: "vsrid_n_{type}" - doc: Shift right and insert - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - static_defs: - - "const N: i32" - attr: - - *neon-stable - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sri, 'N = 2']]}]] - safety: - unsafe: [neon] - types: - - i64 - - u64 - compose: - - FnCall: - - "static_assert!" - - - 'N >= 1 && N <= 64' - - FnCall: - - transmute - - - FnCall: - - "vsri_n_{type}::" - - - FnCall: - - transmute - - - a - - FnCall: - - transmute - - - b - - - name: "vpmaxnm{neon_type.no}" - doc: "Floating-point Maximum Number Pairwise (vector)." - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmaxnmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float32x2_t - - float64x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vpmaxnm{neon_type}" - links: - - link: "llvm.aarch64.neon.fmaxnmp.{neon_type}" - arch: aarch64,arm64ec - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['*mut f64', float64x1x2_t, float64x1_t] - - ['*mut f64', float64x2x2_t, float64x2_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'a']] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['*mut f64', float64x1x3_t, float64x1_t] - - ['*mut f64', float64x2x3_t, float64x2_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "c: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'a']] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['*mut f64', float64x1x4_t, float64x1_t] - - ['*mut f64', float64x2x4_t, float64x2_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "c: {neon_type[2]}" - - "d: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x{neon_type[1].tuple}.{neon_type[2]}.p0f64" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'b.3', 'a']] - - - name: "vfma{type[3]}" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '1', '_lane_f32'] - - [float32x2_t, float32x4_t, '2', '_laneq_f32'] - - [float32x4_t, float32x2_t, '1', 'q_lane_f32'] - - [float32x4_t, float32x4_t, '2', 'q_laneq_f32'] - - [float64x2_t, float64x2_t, '1', 'q_laneq_f64'] - compose: - - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] - - FnCall: - - "vfma{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vfma_lane_f64" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - FnCall: ["static_assert!", ["LANE == 0"]] - - FnCall: - - "vfma{neon_type.no}" - - - a - - b - - FnCall: ["vdup{neon_type.N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vfma_laneq_f64" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x1_t, float64x2_t] - compose: - - FnCall: ["static_assert_uimm_bits!", ["LANE", "1"]] - - FnCall: - - "vfma{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vfmaq_lane_f64" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmla, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x2_t, float64x1_t] - compose: - - FnCall: ["static_assert!", ["LANE == 0"]] - - FnCall: - - "vfma{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]] - - - name: "vfma{type[2]}" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f32", float32x2_t, "s_lane_f32", '1'] - - ["f32", float32x4_t, "s_laneq_f32", '2'] - - ["f64", float64x2_t, "d_laneq_f64", '1'] - compose: - - LLVMLink: - name: "_vfma{type[2]}" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {type[0]}"] - links: - - link: "llvm.fma.{type[0]}" - arch: aarch64,arm64ec - - FnCall: [static_assert_uimm_bits!, ['LANE', "{type[3]}"]] - - Let: [c, "{type[0]}", {FnCall: [simd_extract!, [c, 'LANE as u32']]}] - - FnCall: ["_vfma{type[2]}", [b, c, a]] - - - name: "vfmad_lane_f64" - doc: "Floating-point fused multiply-add to accumulator" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmadd, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f64", float64x1_t] - compose: - - LLVMLink: - name: "_vfmad_lane_f64" - arguments: ["a: f64", "b: f64", "c: f64"] - links: - - link: "llvm.fma.{type[0]}" - arch: aarch64,arm64ec - - FnCall: [static_assert!, ['LANE == 0']] - - Let: [c, "{type[0]}", {FnCall: [simd_extract!, [c, 'LANE as u32']]}] - - FnCall: ["_vfmad_lane_f64", [b, c, a]] - - - name: "vfms_f64" - doc: "Floating-point fused multiply-subtract from accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] - - FnCall: [vfma_f64, [a, b, c]] - - - name: "vfms{neon_type.no}" - doc: "Floating-point fused multiply-subtract from accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x2_t - compose: - - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] - - FnCall: [vfmaq_f64, [a, b, c]] - - - name: "vmls{neon_type.no}" - doc: "Floating-point multiply-subtract from accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - FnCall: [simd_sub, [a, {FnCall: [simd_mul, [b, c]]}]] - - - name: "vfms{type[3]}" - doc: "Floating-point fused multiply-subtract to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '1', _lane_f32] - - [float32x2_t, float32x4_t, '2', _laneq_f32] - - [float32x4_t, float32x2_t, '1', q_lane_f32] - - [float32x4_t, float32x4_t, '2', q_laneq_f32] - - [float64x2_t, float64x2_t, '1', q_laneq_f64] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[2]}']] - - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] - - - name: "vfms_lane_f64" - doc: "Floating-point fused multiply-subtract to accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: ["vfms{neon_type.no}", [a, b, {FnCall: ["vdup{neon_type.N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] - - - name: "vfms_laneq_f64" - doc: "Floating-point fused multiply-subtract to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x1_t, float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] - - - name: "vfmsq_lane_f64" - doc: "Floating-point fused multiply-subtract to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x2_t, float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: ["vfms{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [{FnCall: [simd_extract!, [c, 'LANE as u32']]}]]}]] - - - name: "vfms{type[2]}" - doc: "Floating-point fused multiply-subtract to accumulator" - arguments: ["a: {type[0]}", "b: {type[0]}", "c: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f32", float32x2_t, "s_lane_f32"] - - ["f32", float32x4_t, "s_laneq_f32"] - - ["f64", float64x1_t, "d_lane_f64"] - - ["f64", float64x2_t, "d_laneq_f64"] - compose: - - FnCall: ["vfma{type[2]}::", ['a', '-b', 'c']] - - - name: "vceqz{neon_type[0].no}" - doc: "Floating-point compare bitwise equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmeq]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, 'f32x2', 'f32x2::new(0.0, 0.0)'] - - [float32x4_t, uint32x4_t, 'f32x4', 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] - - [float64x1_t, uint64x1_t, 'f64', '0.0'] - - [float64x2_t, uint64x2_t, 'f64x2', 'f64x2::new(0.0, 0.0)'] - compose: - - Let: [b, '{type[2]}', '{type[3]}'] - - FnCall: [simd_eq, [a, {FnCall: [transmute, [b]]}]] - - - name: "vceqz{type[2]}" - doc: "Floating-point compare bitwise equal to zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", "s_f32"] - - ["f64", "u64", "d_f64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vceqz_{type[0]}" - - - FnCall: ["vdup_n_{type[0]}", [a]] - - '0' - - - name: "vceqzd_{type[2]}" - doc: "Compare bitwise equal to zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64", "s64"] - - ["u64", "u64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - "vceqz_{type[2]}" - - - FnCall: [transmute, [a]] - - - name: "vceqz{neon_type[0].no}" - doc: "Signed compare bitwise equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - - [poly8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [poly8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [poly64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [poly64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_eq - - - a - - FnCall: [transmute, [b]] - - - name: "vceqz{neon_type[0].no}" - doc: "Unsigned compare bitwise equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmeq]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint8x8_t, u8x8, 'u8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint8x16_t, uint8x16_t, u8x16, 'u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint16x4_t, uint16x4_t, u16x4, 'u16x4::new(0, 0, 0, 0)'] - - [uint16x8_t, uint16x8_t, u16x8, 'u16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint32x2_t, uint32x2_t, u32x2, 'u32x2::new(0, 0)'] - - [uint32x4_t, uint32x4_t, u32x4, 'u32x4::new(0, 0, 0, 0)'] - - [uint64x1_t, uint64x1_t, u64x1, 'u64x1::new(0)'] - - [uint64x2_t, uint64x2_t, u64x2, 'u64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_eq - - - a - - FnCall: [transmute, [b]] - - - name: "vcge{neon_type.no}" - doc: "Compare unsigned greater than or equal" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhs]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - uint64x1_t - - uint64x2_t - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vcge{type[0]}" - doc: "Floating-point compare greater than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vcge_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - FnCall: ["vdup_n_{type[1]}", [b]] - - '0' - - - name: "vcge{neon_type[0].no}" - doc: "Floating-point compare greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vcge{type[0]}" - doc: "Compare greater than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["d_s64", "i64", "u64", s64] - - ["d_u64", "u64", "u64", u64] - compose: - - FnCall: - - transmute - - - FnCall: - - "vcge_{type[3]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vclt{neon_type.no}" - doc: "Compare unsigned less than" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhi]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - uint64x1_t - - uint64x2_t - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vcltd_{type[0]}" - doc: "Compare less than" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s64", "i64", "u64"] - - ["u64", "u64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - "vclt_{type[0]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vtst{neon_type[0].no}" - doc: "Unsigned compare bitwise Test bits nonzero" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmtst]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint64x1_t, u64x1, 'u64x1::new(0)'] - - [uint64x2_t, u64x2, 'u64x2::new(0, 0)'] - compose: - - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] - - Let: [d, "{type[1]}", "{type[2]}"] - - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] - - - name: "vcgez{neon_type[0].no}" - doc: "Floating-point compare greater than or equal to zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmge]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] - - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] - - [float64x1_t, uint64x1_t, f64, '0.0'] - - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_ge - - - a - - FnCall: [transmute, [b]] - - - name: "vcgez{type[0]}" - doc: "Floating-point compare greater than or equal to zero" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vcgez_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - '0' - - - name: "vclezd_s64" - doc: "Compare less than or equal to zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: [vclez_s64, [{FnCall: [transmute, [a]]}]] - - - name: "vcgtd_{type[2]}" - doc: "Compare greater than" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64", 's64'] - - ["u64", "u64", 'u64'] - compose: - - FnCall: - - transmute - - - FnCall: - - "vcgt_{type[2]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vcgtz{neon_type[0].no}" - doc: "Compare signed greater than zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [int64x1_t, uint64x1_t, i64x1, 'i64x1::new(0)'] - - [int64x2_t, uint64x2_t, i64x2, 'i64x2::new(0, 0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_gt - - - a - - FnCall: [transmute, [b]] - - - name: "vcgtzd_s64" - doc: "Compare signed greater than zero" - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["i64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - vcgtz_s64 - - - FnCall: [transmute, [a]] - - - name: "vcgtz{neon_type[0].no}" - doc: "Floating-point compare greater than zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, f32x2, 'f32x2::new(0.0, 0.0)'] - - [float32x4_t, uint32x4_t, f32x4, 'f32x4::new(0.0, 0.0, 0.0, 0.0)'] - - [float64x1_t, uint64x1_t, f64, '0.0'] - - [float64x2_t, uint64x2_t, f64x2, 'f64x2::new(0.0, 0.0)'] - compose: - - Let: [b, "{type[2]}", "{type[3]}"] - - FnCall: [simd_gt, [a, {FnCall: [transmute, [b]]}]] - - - name: "vcgtz{type[0]}" - doc: "Floating-point compare greater than zero" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32", "u32"] - - ["d_f64", "f64", "u64"] - compose: - - FnCall: - - "simd_extract!" - - - FnCall: - - "vcgtz_{type[1]}" - - - FnCall: ["vdup_n_{type[1]}", [a]] - - '0' - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vmul{neon_type[0].N}" - doc: "Vector multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, "f64"] - - [float64x2_t, "f64"] - compose: - - FnCall: - - simd_mul - - - a - - FnCall: ["vdup{neon_type[0].N}", [b]] - - - name: "vmul_lane_f64" - doc: "Floating-point multiply" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - float64x1_t - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - simd_mul - - - a - - FnCall: - - "transmute::" - - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - - name: "vmulq_lane_f64" - doc: "Floating-point multiply" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x2_t, float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: - - simd_mul - - - a - - FnCall: ["simd_shuffle!", [b, b, '[LANE as u32, LANE as u32]']] - - - name: "vmuld_lane_f64" - doc: "Floating-point multiply" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f64", float64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Let: [b, '{type[0]}', {FnCall: [simd_extract!, [b, 'LANE as u32']]}] - - Identifier: ['a * b', Symbol] - - - name: "vmul_laneq_f64" - doc: "Floating-point multiply" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x1_t, float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: - - simd_mul - - - a - - FnCall: - - "transmute::" - - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - - name: "vmulq_laneq_f64" - doc: "Floating-point multiply" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float64x2_t, float64x2_t, float64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: - - simd_mul - - - a - - FnCall: [simd_shuffle!, [b, b, '[LANE as u32, LANE as u32]']] - - - name: "vmul{type[2]}" - doc: "Floating-point multiply" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["f32", float32x2_t, "s_lane_f32", '1'] - - ["f32", float32x4_t, "s_laneq_f32", '2'] - - ["f64", float64x2_t, "d_laneq_f64", '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - Let: [b, '{type[0]}', {FnCall: [simd_extract!, [b, 'LANE as u32']]}] - - Identifier: ['a * b', Symbol] - - - name: "vrsrad_n_s64" - doc: "Signed rounding shift right and accumulate." - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - "i64" - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - Let: [b, "{type}", {FnCall: ["vrshrd_n_s64::", [b]]}] - - Identifier: ['a.wrapping_add(b)', Symbol] - - - name: "vmlsl_high_n_{neon_type[1]}" - doc: "Multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, "i16"] - - [int64x2_t, int32x4_t, "i32"] - compose: - - FnCall: ["vmlsl_high_{neon_type[1]}", [a, b, {FnCall: ["vdupq_n_{neon_type[1]}", [c]]}]] - - - name: "vmlsl_high_n_{neon_type[1]}" - doc: "Multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x8_t, "u16"] - - [uint64x2_t, uint32x4_t, "u32"] - compose: - - FnCall: ["vmlsl_high_{neon_type[1]}", [a, b, {FnCall: ["vdupq_n_{neon_type[1]}", [c]]}]] - - - name: "vmlsl_high_lane{neon_type[2].no}" - doc: "Multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlsl2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int64x2_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int64x2_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmlsl_high_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmlsl_high_lane{neon_type[2].no}" - doc: "Multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlsl2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmlsl_high_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vclt{neon_type[0].no}" - doc: "Floating-point compare less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmgt]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vclt{type[2]}" - doc: "Floating-point compare less than" - arguments: ["a: {type[0]}", "b: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["f32", "u32", 's_f32'] - - ["f64", "u64", 'd_f64'] - compose: - - FnCall: - - simd_extract! - - - FnCall: - - "vclt_{type[0]}" - - - FnCall: ["vdup_n_{type[0]}", [a]] - - FnCall: ["vdup_n_{type[0]}", [b]] - - '0' - - - name: "vabdl_high_{neon_type[0]}" - doc: "Unsigned Absolute difference Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uabdl]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint8x16_t, uint16x8_t, uint8x8_t, '[8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x8_t, uint32x4_t, uint16x4_t, '[4, 5, 6, 7]'] - - [uint32x4_t, uint64x2_t, uint32x2_t, '[2, 3]'] - compose: - - Let: [c, "{neon_type[2]}", {FnCall: [simd_shuffle!, [a, a, "{type[3]}"]]}] - - Let: [d, "{neon_type[2]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - FnCall: [simd_cast, [{FnCall: ["vabd_{neon_type[0]}", [c, d]]}]] - - - name: "vfms_n_f64" - doc: "Floating-point fused Multiply-subtract to accumulator(vector)" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmsub]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, "f64"] - compose: - - FnCall: - - "vfms{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [c]] - - - name: "vfmsq_n_f64" - doc: "Floating-point fused Multiply-subtract to accumulator(vector)" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmls]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x2_t, "f64"] - compose: - - FnCall: - - "vfms{neon_type[1].no}" - - - a - - b - - FnCall: ["vdup{neon_type[1].N}", [c]] - - - name: "vpminnm{type[0]}" - doc: "Floating-point minimum number pairwise" - arguments: ["a: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['s_f32', float32x2_t, "f32"] - - ['qd_f64', float64x2_t, "f64"] - compose: - - LLVMLink: - name: "vpminnm{type[0]}" - links: - - link: "llvm.aarch64.neon.fminnmv.{type[2]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vpmaxnm{type[0]}" - doc: "Floating-point maximum number pairwise" - arguments: ["a: {neon_type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fmaxnmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ['s_f32', float32x2_t, "f32"] - - ['qd_f64', float64x2_t, "f64"] - compose: - - LLVMLink: - name: "vpmaxnm{type[0]}" - links: - - link: "llvm.aarch64.neon.fmaxnmv.{type[2]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vcled_{type[0]}" - doc: "Compare less than or equal" - arguments: ["a: {type[1]}", "b: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s64", "i64", "u64"] - - ["u64", "u64", "u64"] - compose: - - FnCall: - - transmute - - - FnCall: - - "vcle_{type[0]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vqdmulh{neon_type[0].lane_nox}" - doc: "Vector saturating doubling multiply high by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqdmulh, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, '2'] - - [int16x8_t, int16x4_t, '2'] - - [int32x2_t, int32x2_t, '1'] - - [int32x4_t, int32x2_t, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - FnCall: - - "vqdmulh{neon_type[0].no}" - - - a - - FnCall: - - "vdup{neon_type[0].N}" - - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - - name: "vqabs{type[2]}" - doc: "Signed saturating absolute value" - arguments: ["a: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] - safety: - unsafe: [neon] - types: - - ["i8", "s8", 'b_s8'] - - ["i16", "s16", 'h_s16'] - compose: - - FnCall: - - "simd_extract!" - - - FnCall: ["vqabs_{type[1]}", [{FnCall: ["vdup_n_{type[1]}", [a]]}]] - - '0' - - - name: "vqabs{type[1]}" - doc: "Signed saturating absolute value" - arguments: ["a: {type[0]}"] - return_type: "{type[0]}" - attr: - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]]}, {FnCall: [assert_instr, [sqabs]]}]] - safety: - unsafe: [neon] - types: - - ["i32", "s_s32"] - - ["i64", "d_s64"] - compose: - - LLVMLink: - name: "vqabs{type[1]}" - links: - - link: "llvm.aarch64.neon.sqabs.{type[0]}" - arch: aarch64,arm64ec - - - name: "vmull_high_n_{neon_type[0]}" - doc: "Multiply long" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smull2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int16x8_t, "i16", int32x4_t] - - [int32x4_t, "i32", int64x2_t] - compose: - - FnCall: - - "vmull_high_{neon_type[0]}" - - - a - - FnCall: ["vdupq_n_{neon_type[0]}", [b]] - - - name: "vmull_high_n_{neon_type[0]}" - doc: "Multiply long" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umull2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint16x8_t, "u16", uint32x4_t] - - [uint32x4_t, "u32", uint64x2_t] - compose: - - FnCall: - - "vmull_high_{neon_type[0]}" - - - a - - FnCall: ["vdupq_n_{neon_type[0]}", [b]] - - - name: "vmull_high_lane{neon_type[1].no}" - doc: "Multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smull2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int16x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int16x8_t, int16x8_t, int32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int32x2_t, int64x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int32x4_t, int64x2_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmull_high_{neon_type[0]}" - - - a - - FnCall: [simd_shuffle!, [b, b, '{type[4]}']] - - - name: "vmull_high_lane{neon_type[1].no}" - doc: "Multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umull2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x8_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmull_high_{neon_type[0]}" - - - a - - FnCall: [simd_shuffle!, [b, b, '{type[4]}']] - - - name: "vrsqrte{neon_type.no}" - doc: "Reciprocal square-root estimate." - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrte]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float64x1_t - - float64x2_t - compose: - - LLVMLink: - name: "vrsqrte{neon_type.no}" - links: - - link: "llvm.aarch64.neon.frsqrte.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrsqrte{type[0]}" - doc: "Reciprocal square-root estimate." - arguments: ["a: {type[1]}"] - return_type: "{type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [frsqrte]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["s_f32", "f32"] - - ["d_f64", "f64"] - compose: - - LLVMLink: - name: "vrsqrte{neon_type[1].no}" - links: - - link: "llvm.aarch64.neon.frsqrte.{type[1]}" - arch: aarch64,arm64ec - - - name: "vpminnm{neon_type.no}" - doc: "Floating-point Minimum Number Pairwise (vector)." - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fminnmp]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - float32x2_t - - float64x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vpminnm{neon_type.no}" - links: - - link: "llvm.aarch64.neon.fminnmp.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqshlu{type[0]}" - doc: "Signed saturating shift left unsigned" - arguments: ["a: {type[1]}"] - return_type: "{type[2]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshlu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [b_n_s8, i8, u8, '3', s8] - - [h_n_s16, i16, u16, '4', s16] - - [s_n_s32, i32, u32, '5', s32] - - [d_n_s64, i64, u64, '6', s64] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: - - simd_extract! - - - FnCall: - - "vqshlu_n_{type[4]}::" - - - FnCall: ["vdup_n_{type[4]}", [a]] - - '0' - - - name: "vcvta{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned integer, rounding to nearest with ties to away" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtau]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - - [float64x1_t, uint64x1_t] - - [float64x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "vcvta{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.fcvtau.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to signed fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [float64x1_t, int64x1_t] - - [float64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vmlal_high_n_{neon_type[1]}" - doc: "Multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, "i16"] - - [int64x2_t, int32x4_t, "i32"] - compose: - - FnCall: - - "vmlal_high_{neon_type[1]}" - - - a - - b - - FnCall: ["vdupq_n_{neon_type[1]}", [c]] - - - name: "vmlal_high_n_{neon_type[1]}" - doc: "Multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x8_t, "u16"] - - [uint64x2_t, uint32x4_t, "u32"] - compose: - - FnCall: - - "vmlal_high_{neon_type[1]}" - - - a - - b - - FnCall: ["vdupq_n_{neon_type[1]}", [c]] - - - name: "vmlal_high_lane{neon_type[2].no}" - doc: "Multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [smlal2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int64x2_t, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int64x2_t, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: ['vmlal_high_{neon_type[2]}', [a, b, {FnCall: [simd_shuffle!, [c, c, '{type[4]}']]}]] - - - name: "vmlal_high_lane{neon_type[2].no}" - doc: "Multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [umlal2, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: ['vmlal_high_{neon_type[2]}', [a, b, {FnCall: [simd_shuffle!, [c, c, '{type[4]}']]}]] - - - name: "vrsrad_n_u64" - doc: "Unsigned rounding shift right and accumulate." - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - "u64" - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 64']] - - Let: [b, u64, {FnCall: ["vrshrd_n_u64::", [b]]}] - - Identifier: ['a.wrapping_add(b)', Symbol] - - - name: "vcle{neon_type.no}" - doc: "Compare unsigned less than or equal" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [cmhs]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - uint64x1_t - - uint64x2_t - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x2x4_t, "v2i64", "p0i64"] - - ["*const f64", float64x1x4_t, "v1f64", "p0f64"] - - ["*const f64", float64x2x4_t, "v2f64", "p0f64"] - compose: - - LLVMLink: - name: "vld4{neon_type[1].dup_nox}" - arguments: - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.ld4r.{type[2]}.{type[3]}" - arch: aarch64,arm64ec - - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as _']] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x2x4_t, "q_dup_s64"] - compose: - - FnCall: - - transmute - - - FnCall: ["vld4{type[2]}", [{FnCall: [transmute, [a]]}]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x2x4_t, "q_dup_s64"] - compose: - - FnCall: - - transmute - - - FnCall: ["vld4{type[2]}", [{FnCall: [transmute, [a]]}]] - diff --git a/library/stdarch/crates/stdarch-gen2/spec/neon/arm_shared.spec.yml b/library/stdarch/crates/stdarch-gen2/spec/neon/arm_shared.spec.yml deleted file mode 100644 index 87559963ed1..00000000000 --- a/library/stdarch/crates/stdarch-gen2/spec/neon/arm_shared.spec.yml +++ /dev/null @@ -1,9789 +0,0 @@ -arch_cfgs: - - arch_name: aarch64 - target_feature: [neon] - llvm_prefix: llvm.aarch64.neon - -# Repeatedly used anchors -# #[stable(feature = "neon_intrinsics", since = "1.59.0")] -neon-stable: &neon-stable - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - -# #[unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800")] -neon-unstable: &neon-unstable - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - -# #[cfg_attr(target_arch = "arm", target_feature(enable = "v7"))] -neon-v7: &neon-v7 - FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v7"']]} ]] - -# #[target_feature(enable = "neon,v7")] -enable-v7: &enable-v7 - FnCall: [target_feature, ['enable = "neon,v7"']] - -# #[cfg_attr(target_arch = "arm", target_feature(enable = "v8"))] -neon-v8: &neon-v8 - FnCall: [cfg_attr, ['target_arch = "arm"', { FnCall: [target_feature, [ 'enable = "v8"']]} ]] - -target-is-arm: &target-is-arm - FnCall: [cfg, ['target_arch = "arm"']] - -# #[cfg(not(target_arch = "arm"))] -target-not-arm: &target-not-arm - FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm"']]}]] - -neon-target-aarch64-arm64ec: &neon-target-aarch64-arm64ec - FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]] - -# #[cfg_attr(not(target_arch = "arm"), stable(feature = "neon_intrinsics", since = "1.59.0"))] -neon-stable-not-arm: &neon-stable-not-arm - FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']]}, *neon-stable]] - -#[cfg_attr(target_arch = "arm", unstable(feature = "stdarch_arm_neon_intrinsics", issue = "111800"))] -neon-unstable-is-arm: &neon-unstable-is-arm - FnCall: [ cfg_attr, ['target_arch = "arm"', *neon-unstable]] - -# #[cfg_attr(all(test, not(target_env = "msvc"))] -msvc-disabled: &msvc-disabled - FnCall: [all, [test, {FnCall: [not, ['target_env = "msvc"']]}]] - -# all(test, target_arch = "arm") -test-is-arm: &test-is-arm - FnCall: [all, [test, 'target_arch = "arm"']] - -# #[target_feature(enable = "neon,aes")] -neon-aes: &neon-aes - FnCall: [target_feature, ['enable = "neon,aes"']] - -# #[target_feature(enable = "neon,i8mm")] -neon-i8mm: &neon-i8mm - FnCall: [target_feature, ['enable = "neon,i8mm"']] - -#[cfg_attr(not(target_arch = "arm"), unstable(feature = "stdarch_neon_i8mm", issue = "117223"))] -neon-unstable-i8mm: &neon-unstable-i8mm - FnCall: [cfg_attr, [{ FnCall: [not, ['target_arch = "arm"']] }, { FnCall: [unstable, ['feature = "stdarch_neon_i8mm"', 'issue = "117223"']] } ]] - -# #[unstable(feature = "stdarch_neon_fcma", issue = "117222")] -neon-unstable-fcma: &neon-unstable-fcma - FnCall: [unstable, ['feature = "stdarch_neon_fcma"', 'issue = "117222"']] - -intrinsics: - - name: "vand{neon_type.no}" - doc: Vector bitwise and - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vand]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [and]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - - int64x1_t - - int64x2_t - - uint64x1_t - - uint64x2_t - compose: - - FnCall: - - simd_and - - - a - - b - - - name: "vorr{neon_type.no}" - doc: "Vector bitwise or (immediate, inclusive)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vorr]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [orr]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - - int64x1_t - - int64x2_t - - uint64x1_t - - uint64x2_t - compose: - - FnCall: - - simd_or - - - a - - b - - - name: "veor{neon_type.no}" - doc: Vector bitwise exclusive or (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [veor]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [eor]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - - int64x1_t - - int64x2_t - - uint64x1_t - - uint64x2_t - compose: - - FnCall: - - simd_xor - - - a - - b - - - name: "vabd{neon_type.no}" - doc: Absolute difference between the arguments - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sabd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "sabd.{neon_type}" - links: - - link: "llvm.aarch64.neon.sabd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vabds.{neon_type}" - arch: arm - - - name: "vabd{neon_type.no}" - doc: Absolute difference between the arguments - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uabd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "uabd.{neon_type}" - links: - - link: "llvm.aarch64.neon.uabd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vabdu.{neon_type}" - arch: arm - - - name: "vabd{neon_type.no}" - doc: Absolute difference between the arguments of Floating - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vabd.f32"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fabd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "fabd.{neon_type}" - links: - - link: "llvm.arm.neon.vabds.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.fabd.{neon_type}" - arch: aarch64,arm64ec - - - name: "vabdl{neon_type[0].noq}" - doc: Signed Absolute difference Long - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - target_feature - - - 'enable = "v7"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - '"vabdl.{neon_type[0]}"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - sabdl - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, uint8x8_t] - - [int16x4_t, int32x4_t, uint16x4_t] - - [int32x2_t, int64x2_t, uint32x2_t] - compose: - - Let: - - c - - "{neon_type[2]}" - - FnCall: - - simd_cast - - - FnCall: - - "vabd_{neon_type[0]}" - - - a - - b - - FnCall: - - simd_cast - - - c - - - name: "vceq{neon_type[0].no}" - doc: "Compare bitwise Equal (vector)" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vceq{type[2]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmeq]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint8x8_t, ".i8"] - - [uint8x16_t, uint8x16_t, ".i8"] - - [int8x8_t, uint8x8_t, ".i8"] - - [int8x16_t, uint8x16_t, ".i8"] - - [poly8x8_t, uint8x8_t, ".i8"] - - [poly8x16_t, uint8x16_t, ".i8"] - - [uint16x4_t, uint16x4_t, ".i16"] - - [uint16x8_t, uint16x8_t, ".i16"] - - [int16x4_t, uint16x4_t, ".i16"] - - [int16x8_t, uint16x8_t, ".i16"] - - [uint32x2_t, uint32x2_t, ".i32"] - - [uint32x4_t, uint32x4_t, ".i32"] - - [int32x2_t, uint32x2_t, ".i32"] - - [int32x4_t, uint32x4_t, ".i32"] - compose: - - FnCall: [simd_eq, [a, b]] - - - name: "vceq{neon_type[0].no}" - doc: "Floating-point compare equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vceq.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmeq]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: [simd_eq, [a, b]] - - - name: "vtst{neon_type[0].no}" - doc: "Signed compare bitwise Test bits nonzero" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtst]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmtst]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [int16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [int16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [int32x2_t, uint32x2_t, i32x2, 'i32x2::new(0, 0)'] - - [int32x4_t, uint32x4_t, i32x4, 'i32x4::new(0, 0, 0, 0)'] - - [poly8x8_t, uint8x8_t, i8x8, 'i8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [poly8x16_t, uint8x16_t, i8x16, 'i8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [poly16x4_t, uint16x4_t, i16x4, 'i16x4::new(0, 0, 0, 0)'] - - [poly16x8_t, uint16x8_t, i16x8, 'i16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - compose: - - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] - - Let: [d, "{type[2]}", "{type[3]}"] - - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] - - - name: "vabs{neon_type.no}" - doc: "Floating-point absolute value" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vabs]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fabs]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - FnCall: [simd_fabs, [a]] - - - name: "vcgt{neon_type[0].no}" - doc: "Compare signed greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{type[2]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, "s8"] - - [int8x16_t, uint8x16_t, "s8"] - - [int16x4_t, uint16x4_t, s16] - - [int16x8_t, uint16x8_t, s16] - - [int32x2_t, uint32x2_t, "s32"] - - [int32x4_t, uint32x4_t, "s32"] - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vcgt{neon_type.no}" - doc: "Compare unsigned greater than" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhi]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vcgt{neon_type[0].no}" - doc: "Floating-point compare greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: [simd_gt, [a, b]] - - - name: "vclt{neon_type[0].no}" - doc: "Compare signed less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type[0]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t] - - [int8x16_t, uint8x16_t] - - [int16x4_t, uint16x4_t] - - [int16x8_t, uint16x8_t] - - [int32x2_t, uint32x2_t] - - [int32x4_t, uint32x4_t] - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vcle{neon_type[0].no}" - doc: "Compare signed less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type[0]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t] - - [int8x16_t, uint8x16_t] - - [int16x4_t, uint16x4_t] - - [int16x8_t, uint16x8_t] - - [int32x2_t, uint32x2_t] - - [int32x4_t, uint32x4_t] - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vcle{neon_type[0].no}" - doc: "Floating-point compare less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vcge{neon_type[0].no}" - doc: "Compare signed greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type[0]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t] - - [int8x16_t, uint8x16_t] - - [int16x4_t, uint16x4_t] - - [int16x8_t, uint16x8_t] - - [int32x2_t, uint32x2_t] - - [int32x4_t, uint32x4_t] - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vcls{neon_type.no}" - doc: "Count leading sign bits" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcls.{neon_type}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "vcls{neon_type.no}" - links: - - link: "llvm.arm.neon.vcls.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.cls.{neon_type}" - arch: aarch64,arm64ec - - - name: "vcls{neon_type[0].no}" - doc: "Count leading sign bits" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcls]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [uint16x4_t, int16x4_t] - - [uint16x8_t, int16x8_t] - - [uint32x2_t, int32x2_t] - - [uint32x4_t, int32x4_t] - compose: - - FnCall: - - "vcls{neon_type[1].no}" - - - FnCall: [transmute, [a]] - - - name: "vclz{neon_type.no}" - doc: "Count leading zero bits" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz.i8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - compose: - - FnCall: ["vclz{neon_type.no}_", [a]] - - - name: "vclz{neon_type[0].no}" - doc: "Count leading zero bits" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz.i8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vclz{neon_type[1].no}_" - - - FnCall: [transmute, [a]] - - - name: "vclz{neon_type[0].no}" - doc: "Count leading zero bits" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz{type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, '.i16'] - - [int16x8_t, '.i16'] - - [int32x2_t, '.i32'] - - [int32x4_t, '.i32'] - compose: - - FnCall: ["vclz{neon_type[0].no}_", [a]] - - - name: "vclz{neon_type[0].no}" - doc: "Count leading zero bits" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vclz{type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [clz]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint32x2_t, '.i32', int32x2_t] - - [uint32x4_t, '.i32', int32x4_t] - - [uint16x4_t, '.i16', int16x4_t] - - [uint16x8_t, '.i16', int16x8_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vclz{neon_type[2].no}_" - - - FnCall: [transmute, [a]] - - - name: "vcagt{neon_type[0].no}" - doc: "Floating-point absolute compare greater than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacgt.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - LLVMLink: - name: "vcagt{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vacgt.{neon_type[1]}.{neon_type[0]}" - arch: arm - - link: "llvm.aarch64.neon.facgt.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcage{neon_type[0].no}" - doc: "Floating-point absolute compare greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacge.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - LLVMLink: - name: "vcage{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vacge.{neon_type[1]}.{neon_type[0]}" - arch: arm - - link: "llvm.aarch64.neon.facge.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vcalt{neon_type[0].no}" - doc: "Floating-point absolute compare less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacgt.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: ["vcagt{neon_type[0].no}", [b, a]] - - - name: "vcale{neon_type[0].no}" - doc: "Floating-point absolute compare less than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vacge.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [facge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: ["vcage{neon_type[0].no}", [b, a]] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [scvtf]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, float32x2_t] - - [int32x4_t, float32x4_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ucvtf]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint32x2_t, float32x2_t] - - [uint32x4_t, float32x4_t] - compose: - - FnCall: [simd_cast, [a]] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint32x2_t, float32x2_t] - - [uint32x4_t, float32x4_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.arm.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" - arch: arm - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, float32x2_t] - - [int32x4_t, float32x4_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.arm.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" - arch: arm - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [scvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, float32x2_t] - - [int32x4_t, float32x4_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxs2fp.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", [a, N]] - - - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - doc: "Fixed-point convert to floating-point" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ucvtf, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint32x2_t, float32x2_t] - - [uint32x4_t, float32x4_t] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{neon_type[1].N}_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfxu2fp.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{neon_type[1].N}_{neon_type[0]}", ["a.as_signed()", N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t, _n_s32_f32] - - [float32x4_t, int32x4_t, q_n_s32_f32] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.arm.neon.vcvtfp2fxs.{neon_type[1]}.{neon_type[0]}" - arch: arm - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vcvt, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, _n_u32_f32] - - [float32x4_t, uint32x4_t, q_n_u32_f32] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.arm.neon.vcvtfp2fxu.{neon_type[1]}.{neon_type[0]}" - arch: arm - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzs, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t, _n_s32_f32] - - [float32x4_t, int32x4_t, q_n_s32_f32] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: ["a: {type[0]}", "n: i32"] - links: - - link: "llvm.aarch64.neon.vcvtfp2fxs.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vcvt{type[2]}" - doc: "Floating-point convert to fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [fcvtzu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t, _n_u32_f32] - - [float32x4_t, uint32x4_t, q_n_u32_f32] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= 32']] - - LLVMLink: - name: "vcvt{type[2]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.vcvtfp2fxu.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vcvt{type[2]}", [a, N]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.8"', 'N = 4']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 4']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s8, int8x8_t, int8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_s8, int8x8_t, int8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_lane_u8, uint8x8_t, uint8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_u8, uint8x8_t, uint8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_lane_p8, poly8x8_t, poly8x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_p8, poly8x8_t, poly8x16_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.8"', 'N = 8']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 8']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [q_laneq_s8, int8x16_t, int8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_s8, int8x16_t, int8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [q_laneq_u8, uint8x16_t, uint8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_u8, uint8x16_t, uint8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [q_laneq_p8, poly8x16_t, poly8x16_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_p8, poly8x16_t, poly8x8_t, '4', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.16"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_s16, int16x4_t, int16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_u16, uint16x4_t, uint16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_lane_p16, poly16x4_t, poly16x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [q_lane_p16, poly16x4_t, poly16x8_t, '2', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.16"', 'N = 4']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 4']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_s16, int16x8_t, int16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] - - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_u16, uint16x8_t, uint16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] - - [q_laneq_p16, poly16x8_t, poly16x8_t, '3', '[N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_p16, poly16x8_t, poly16x4_t, '3', '[N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.32"', 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s32, int32x2_t, int32x2_t, '1', '[N as u32, N as u32]'] - - [q_lane_s32, int32x2_t, int32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] - - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[N as u32, N as u32]'] - - [q_lane_u32, uint32x2_t, uint32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] - - [_lane_f32, float32x2_t, float32x2_t, '1', '[N as u32, N as u32]'] - - [q_lane_f32, float32x2_t, float32x4_t, '1', '[N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vdup.32"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_s32, int32x4_t, int32x2_t, '2', '[N as u32, N as u32]'] - - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_u32, uint32x4_t, uint32x2_t, '2', '[N as u32, N as u32]'] - - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[N as u32, N as u32, N as u32, N as u32]'] - - [_laneq_f32, float32x4_t, float32x2_t, '2', '[N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[3]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[4]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [q_laneq_s64, int64x2_t, '1', '[N as u32, N as u32]'] - - [q_laneq_u64, uint64x2_t, '1', '[N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] - - FnCall: [simd_shuffle!, [a, a, "{type[3]}"]] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [dup, 'N = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [q_lane_s64, int64x1_t, int64x2_t] - - [q_lane_u64, uint64x1_t, uint64x2_t] - compose: - - FnCall: [static_assert!, ['N == 0']] - - FnCall: [simd_shuffle!, [a, a, '[N as u32, N as u32]']] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'N = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'N = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s64, int64x1_t] - - [_lane_u64, uint64x1_t] - compose: - - FnCall: [static_assert!, ['N == 0']] - - Identifier: [a, Symbol] - - - name: "vdup{type[0]}" - doc: "Set all vector lanes to the same value" - arguments: ["a: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [_laneq_s64, int64x2_t, int64x1_t, '::'] - - [_laneq_u64, uint64x2_t, uint64x1_t, '::'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, 1]] - - FnCall: - - "transmute{type[3]}" - - - FnCall: [simd_extract!, [a, 'N as u32']] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 7']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 7']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - - [int16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - - [uint8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - - [uint16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - - [poly8x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - - [poly16x8_t, ' static_assert_uimm_bits!(N, 3); match N & 0b111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 15']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 15']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] - - [uint8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] - - [poly8x16_t, ' static_assert_uimm_bits!(N, 4); match N & 0b1111 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]), 4 => simd_shuffle!(a, b, [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]), 5 => simd_shuffle!(a, b, [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]), 6 => simd_shuffle!(a, b, [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]), 7 => simd_shuffle!(a, b, [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22]), 8 => simd_shuffle!(a, b, [8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]), 9 => simd_shuffle!(a, b, [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]), 10 => simd_shuffle!(a, b, [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]), 11 => simd_shuffle!(a, b, [11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26]), 12 => simd_shuffle!(a, b, [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]), 13 => simd_shuffle!(a, b, [13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]), 14 => simd_shuffle!(a, b, [14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]), 15 => simd_shuffle!(a, b, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 3']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 3']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, 'static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - - [int32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - - [uint16x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - - [uint32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - - [poly16x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - - [float32x4_t, ' static_assert_uimm_bits!(N, 2); match N & 0b11 { 0 => simd_shuffle!(a, b, [0, 1, 2, 3]), 1 => simd_shuffle!(a, b, [1, 2, 3, 4]), 2 => simd_shuffle!(a, b, [2, 3, 4, 5]), 3 => simd_shuffle!(a, b, [3, 4, 5, 6]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vext.8"', 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - - [uint32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - - [float32x2_t, ' static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vext{neon_type[0].no}" - doc: "Extract vector from pair of vectors" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmov, 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ext, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, 'static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - - [uint64x2_t, 'static_assert_uimm_bits!(N, 1); match N & 0b1 { 0 => simd_shuffle!(a, b, [0, 1]), 1 => simd_shuffle!(a, b, [1, 2]), _ => unreachable_unchecked(), }'] - compose: - - Identifier: ["{type[1]}", Symbol] - - - name: "vmla{neon_type[0].no}" - doc: "Multiply-add to accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla{type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, ".i8"] - - [int8x16_t, ".i8"] - - [uint8x8_t, ".i8"] - - [uint8x16_t, ".i8"] - - [int16x4_t, ".i16"] - - [int16x8_t, ".i16"] - - [uint16x4_t, ".i16"] - - [uint16x8_t, ".i16"] - - [int32x2_t, ".i32"] - - [int32x4_t, ".i32"] - - [uint32x2_t, ".i32"] - - [uint32x4_t, ".i32"] - compose: - - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] - - - name: "vmla{neon_type.no}" - doc: "Floating-point multiply-add to accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - FnCall: [simd_add, [a, {FnCall: [simd_mul, [b, c]]}]] - - - name: "vmlal{neon_type[1].no}" - doc: "Signed multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[2]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, "s8"] - - [int32x4_t, int16x4_t, "s16"] - - [int64x2_t, int32x2_t, "s32"] - compose: - - FnCall: [simd_add, [a, {FnCall: ["vmull_{type[2]}", [b, c]]}]] - - - name: "vmlal_n_{type[4]}" - doc: "Vector widening multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[4]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, "i16", int32x4_t, 's16'] - - [int64x2_t, int32x2_t, "i32", int64x2_t, 's32'] - compose: - - FnCall: - - "vmlal{neon_type[1].noq}" - - - a - - b - - FnCall: ["vdup_n_{neon_type[1]}", [c]] - - - name: "vmlal_n_{type[2]}" - doc: "Vector widening multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{type[2]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x4_t, "u16", uint32x4_t] - - [uint64x2_t, uint32x2_t, "u32", uint64x2_t] - compose: - - FnCall: - - "vmlal{neon_type[1].noq}" - - - a - - b - - FnCall: ["vdup_n_{neon_type[1]}", [c]] - - - name: "vmlal_lane{neon_type[2].no}" - doc: "Vector widening multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlal, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int64x2_t, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [int64x2_t, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmlal_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] - - - name: "vmlal_lane{neon_type[2].no}" - doc: "Vector widening multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x4_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint16x4_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x2_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x2_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] - - FnCall: - - "vmlal_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, '{type[5]}']] - - - name: "vmlal_{neon_type[1]}" - doc: "Unsigned multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlal.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t] - - [uint32x4_t, uint16x4_t] - - [uint64x2_t, uint32x2_t] - compose: - - FnCall: - - simd_add - - - a - - FnCall: ["vmull_{neon_type[1]}", [b, c]] - - - name: "vmls{neon_type[0].no}" - doc: "Multiply-subtract from accumulator" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls{type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, '.i8'] - - [int8x16_t, '.i8'] - - [uint8x8_t, '.i8'] - - [uint8x16_t, '.i8'] - - [int16x4_t, ".i16"] - - [int16x8_t, ".i16"] - - [uint16x4_t, ".i16"] - - [uint16x8_t, ".i16"] - - [int32x2_t, ".i32"] - - [int32x4_t, ".i32"] - - [uint32x2_t, ".i32"] - - [uint32x4_t, ".i32"] - compose: - - FnCall: - - simd_sub - - - a - - FnCall: [simd_mul, [b, c]] - - - name: "vmlsl_{neon_type[1]}" - doc: "Signed multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t] - - [int32x4_t, int16x4_t] - - [int64x2_t, int32x2_t] - compose: - - FnCall: [simd_sub, [a, {FnCall: ["vmull_{neon_type[1]}", [b, c]]}]] - - - name: "vmlsl_n_{neon_type[1]}" - doc: "Vector widening multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, "i16"] - - [int64x2_t, int32x2_t, "i32"] - compose: - - FnCall: ["vmlsl_{neon_type[1]}", [a, b, {FnCall: ["vdup_n_{neon_type[1]}", [c]]}]] - - - name: "vmlsl_n_{neon_type[1]}" - doc: "Vector widening multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x4_t, "u16"] - - [uint64x2_t, uint32x2_t, "u32"] - compose: - - FnCall: ["vmlsl_{neon_type[1]}", [a, b, {FnCall: ["vdup_n_{neon_type[1]}", [c]]}]] - - - name: "vmlsl_lane{neon_type[2].no}" - doc: "Vector widening multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x4_t, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmlsl_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmlsl_lane{neon_type[2].no}" - doc: "Vector widening multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smlsl, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int32x2_t, int32x2_t, '[LANE as u32, LANE as u32]', '1'] - - [int64x2_t, int32x2_t, int32x4_t, '[LANE as u32, LANE as u32]', '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[4]}"]] - - FnCall: - - "vmlsl_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - - name: "vmlsl_lane{neon_type[2].no}" - doc: "Vector widening multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint32x4_t, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x4_t, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [uint64x2_t, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmlsl_{neon_type[1]}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmlsl_{neon_type[1]}" - doc: "Unsigned multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmlsl.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t] - - [uint32x4_t, uint16x4_t] - - [uint64x2_t, uint32x2_t] - compose: - - FnCall: [simd_sub, [a, {FnCall: ["vmull_{neon_type[1]}", [b, c]]}]] - - - name: "vneg{neon_type[0].no}" - doc: Negate - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vneg.{type[1]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [neg]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, 's8'] - - [int8x16_t, 's8'] - - [int16x4_t, 's16'] - - [int16x8_t, 's16'] - - [int32x2_t, 's32'] - - [int32x4_t, 's32'] - compose: - - FnCall: [simd_neg, [a]] - - - name: "vneg{neon_type[0].no}" - doc: Negate - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vneg.{type[1]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fneg]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [float32x2_t, 'f32'] - - [float32x4_t, 'f32'] - compose: - - FnCall: [simd_neg, [a]] - - - name: "vqneg{neon_type[0].no}" - doc: Signed saturating negate - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqneg.{type[1]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqneg]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, 's8', 'i8'] - - [int8x16_t, 's8', 'i8'] - - [int16x4_t, 's16', 'i16'] - - [int16x8_t, 's16', 'i16'] - - [int32x2_t, 's32', 'i32'] - - [int32x4_t, 's32', 'i32'] - compose: - - LLVMLink: - name: "sqneg.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.sqneg.v{neon_type[0].lane}{type[2]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vqneg.v{neon_type[0].lane}{type[2]}" - arch: arm - - - name: "vqsub{neon_type[0].no}" - doc: Saturating subtract - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqsub.{type[1]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uqsub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint8x8_t, u8, i8] - - [uint8x16_t, u8, i8] - - [uint16x4_t, u16, i16] - - [uint16x8_t, u16, i16] - - [uint32x2_t, u32, i32] - - [uint32x4_t, u32, i32] - - [uint64x1_t, u64, i64] - - [uint64x2_t, u64, i64] - compose: - - LLVMLink: - name: "uqsub.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.uqsub.v{neon_type[0].lane}{type[2]}" - arch: aarch64,arm64ec - - link: "llvm.usub.sat.{neon_type[0].lane}{type[2]}" - arch: arm - - - name: "vqsub{neon_type[0].no}" - doc: Saturating subtract - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqsub.{type[1]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqsub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, s8, i8] - - [int8x16_t, s8, i8] - - [int16x4_t, s16, i16] - - [int16x8_t, s16, i16] - - [int32x2_t, s32, i32] - - [int32x4_t, s32, i32] - - [int64x1_t, s64, i64] - - [int64x2_t, s64, i64] - compose: - - LLVMLink: - name: "sqsub.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.sqsub.v{neon_type[0].lane}{type[2]}" - arch: aarch64,arm64ec - - link: "llvm.ssub.sat.{neon_type[0].lane}{type[2]}" - arch: arm - - - name: "vhadd{neon_type.no}" - doc: Halving add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - target_feature - - - 'enable = "v7"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - '"vhadd.{neon_type}"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - uhadd - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "uhadd.{neon_type}" - links: - - link: "llvm.aarch64.neon.uhadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vhaddu.{neon_type}" - arch: arm - - - name: "vhadd{neon_type.no}" - doc: Halving add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - target_feature - - - 'enable = "v7"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - '"vhadd.{neon_type}"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - shadd - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "shadd.{neon_type}" - links: - - link: "llvm.aarch64.neon.shadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vhadds.{neon_type}" - arch: arm - - - name: "vrhadd{neon_type.no}" - doc: Rounding halving add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vrhadd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [srhadd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "vrhadd.{neon_type}" - links: - - link: "llvm.aarch64.neon.srhadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vrhadds.{neon_type}" - arch: arm - - - name: "vrhadd{neon_type.no}" - doc: Rounding halving add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vrhadd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [urhadd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "vrhaddu.{neon_type}" - links: - - link: "llvm.aarch64.neon.urhadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vrhaddu.{neon_type}" - arch: arm - - - name: "vrndn{neon_type.no}" - doc: "Floating-point round to integral, to nearest with ties to even" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vrintn]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frintn]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "llvm.frinn.{neon_type}" - links: - - link: "llvm.aarch64.neon.frintn.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vrintn.{neon_type}" - arch: arm - - - name: "vqadd{neon_type.no}" - doc: Saturating add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqadd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uqadd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - - uint64x1_t - - uint64x2_t - compose: - - LLVMLink: - name: "uqadd.{neon_type}" - links: - - link: "llvm.aarch64.neon.uqadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.uadd.sat.{neon_type}" - arch: arm - - - name: "vqadd{neon_type.no}" - doc: Saturating add - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqadd.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqadd]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "sqadd.{neon_type}" - links: - - link: "llvm.aarch64.neon.sqadd.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.sadd.sat.{neon_type}" - arch: arm - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const f32", float32x2x2_t] - - ["*const f32", float32x4x2_t] - - ["*const f32", float32x2x3_t] - - ["*const f32", float32x4x3_t] - - ["*const f32", float32x2x4_t] - - ["*const f32", float32x4x4_t] - compose: - - LLVMLink: - name: "vld1x{neon_type[1].tuple}.{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f{neon_type[1].base}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vld1x{neon_type[1].tuple}.v{neon_type[1].lane}f{neon_type[1].base}.p0f{neon_type[1].base}" - arch: arm - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t] - - ["*const i8", int8x16x2_t] - - ["*const i8", int8x8x3_t] - - ["*const i8", int8x16x3_t] - - ["*const i8", int8x8x4_t] - - ["*const i8", int8x16x4_t] - - ["*const i16", int16x4x2_t] - - ["*const i16", int16x8x2_t] - - ["*const i16", int16x4x3_t] - - ["*const i16", int16x8x3_t] - - ["*const i16", int16x4x4_t] - - ["*const i16", int16x8x4_t] - - ["*const i32", int32x2x2_t] - - ["*const i32", int32x4x2_t] - - ["*const i32", int32x2x3_t] - - ["*const i32", int32x4x3_t] - - ["*const i32", int32x2x4_t] - - ["*const i32", int32x4x4_t] - - ["*const i64", int64x1x2_t] - - ["*const i64", int64x1x3_t] - - ["*const i64", int64x1x4_t] - - ["*const i64", int64x2x2_t] - - ["*const i64", int64x2x3_t] - - ["*const i64", int64x2x4_t] - compose: - - LLVMLink: - name: "ld1x{neon_type[1].tuple}.{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.ld1x{neon_type[1].tuple}.v{neon_type[1].lane}i{neon_type[1].base}.p0i{neon_type[1].base}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vld1x{neon_type[1].tuple}.v{neon_type[1].lane}i{neon_type[1].base}.p0i{neon_type[1].base}" - arch: arm - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x8x2_t, int8x8x2_t] - - ["*const u8", uint8x16x2_t, int8x16x2_t] - - ["*const u8", uint8x8x3_t, int8x8x3_t] - - ["*const u8", uint8x16x3_t, int8x16x3_t] - - ["*const u8", uint8x8x4_t, int8x8x4_t] - - ["*const u8", uint8x16x4_t, int8x16x4_t] - - ["*const u16", uint16x4x2_t, int16x4x2_t] - - ["*const u16", uint16x8x2_t, int16x8x2_t] - - ["*const u16", uint16x4x3_t, int16x4x3_t] - - ["*const u16", uint16x8x3_t, int16x8x3_t] - - ["*const u16", uint16x4x4_t, int16x4x4_t] - - ["*const u16", uint16x8x4_t, int16x8x4_t] - - ["*const u32", uint32x2x2_t, int32x2x2_t] - - ["*const u32", uint32x4x2_t, int32x4x2_t] - - ["*const u32", uint32x2x3_t, int32x2x3_t] - - ["*const u32", uint32x4x3_t, int32x4x3_t] - - ["*const u32", uint32x2x4_t, int32x2x4_t] - - ["*const u32", uint32x4x4_t, int32x4x4_t] - - ["*const u64", uint64x1x2_t, int64x1x2_t] - - ["*const u64", uint64x1x3_t, int64x1x3_t] - - ["*const u64", uint64x1x4_t, int64x1x4_t] - - ["*const u64", uint64x2x2_t, int64x2x2_t] - - ["*const u64", uint64x2x3_t, int64x2x3_t] - - ["*const u64", uint64x2x4_t, int64x2x4_t] - - ["*const p8", poly8x8x2_t, int8x8x2_t] - - ["*const p8", poly8x8x3_t, int8x8x3_t] - - ["*const p8", poly8x8x4_t, int8x8x4_t] - - ["*const p8", poly8x16x2_t, int8x16x2_t] - - ["*const p8", poly8x16x3_t, int8x16x3_t] - - ["*const p8", poly8x16x4_t, int8x16x4_t] - - ["*const p16", poly16x4x2_t, int16x4x2_t] - - ["*const p16", poly16x4x3_t, int16x4x3_t] - - ["*const p16", poly16x4x4_t, int16x4x4_t] - - ["*const p16", poly16x8x2_t, int16x8x2_t] - - ["*const p16", poly16x8x3_t, int16x8x3_t] - - ["*const p16", poly16x8x4_t, int16x8x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld1{neon_type[2].no}" - - - FnCall: - - transmute - - - a - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x3_t, int64x1x3_t] - - ["*const p64", poly64x1x4_t, int64x1x4_t] - - ["*const p64", poly64x2x2_t, int64x2x2_t] - - ["*const p64", poly64x2x3_t, int64x2x3_t] - - ["*const p64", poly64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld1{neon_type[2].no}" - - - FnCall: - - transmute - - - a - - - name: "vld1{neon_type[1].no}" - doc: "Load multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld1]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld1{neon_type[2].no}" - - - FnCall: - - transmute - - - a - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [vld2] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8] - - ["*const i16", int16x4x2_t, i16] - - ["*const i32", int32x2x2_t, i32] - - ["*const i8", int8x16x2_t, i8] - - ["*const i16", int16x8x2_t, i16] - - ["*const i32", int32x4x2_t, i32] - - ["*const f32", float32x2x2_t, f32] - - ["*const f32", float32x4x2_t, f32] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as *const i8" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x2_t, i64] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as *const i8" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - *neon-stable - assert_instr: [ld2] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8, int8x8_t] - - ["*const i16", int16x4x2_t, i16, int16x4_t] - - ["*const i32", int32x2x2_t, i32, int32x2_t] - - ["*const i8", int8x16x2_t, i8, int8x16_t] - - ["*const i16", int16x8x2_t, i16, int16x8_t] - - ["*const i32", int32x4x2_t, i32, int32x4_t] - - ["*const f32", float32x2x2_t, f32, float32x2_t] - - ["*const f32", float32x4x2_t, f32, float32x4_t] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const {neon_type[3]}" - links: - - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as _" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - *neon-stable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x2_t, i64, int64x1_t] - compose: - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const {neon_type[3]}" - links: - - link: "llvm.aarch64.neon.ld2.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].nox}" - - - "a as _" - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x8x2_t, int8x8x2_t] - - ["*const u16", uint16x4x2_t, int16x4x2_t] - - ["*const u32", uint32x2x2_t, int32x2x2_t] - - ["*const u8", uint8x16x2_t, int8x16x2_t] - - ["*const u16", uint16x8x2_t, int16x8x2_t] - - ["*const u32", uint32x4x2_t, int32x4x2_t] - - ["*const p8", poly8x8x2_t, int8x8x2_t] - - ["*const p16", poly16x4x2_t, int16x4x2_t] - - ["*const p8", poly8x16x2_t, int8x16x2_t] - - ["*const p16", poly16x8x2_t, int16x8x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - - name: "vld2{neon_type[1].nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - FnCall: - - cfg_attr - - - test - - FnCall: - - assert_instr - - - vld2 - - "LANE = 0" - - FnCall: - - rustc_legacy_const_generics - - - "2" - - *neon-unstable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8, int8x8_t, "3"] - - ["*const i16", int16x4x2_t, i16, int16x4_t, "2"] - - ["*const i32", int32x2x2_t, i32, int32x2_t, "1"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[4]}" - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "a: {neon_type[3]}" - - "b: {neon_type[3]}" - - "n: i32" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2lane.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2_lane{neon_type[1].nox}" - - - "a as _" - - "b.0" - - "b.1" - - "LANE" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x8x2_t, int8x8_t, "3"] - - ["*const u16", uint16x4x2_t, int16x4_t, "2"] - - ["*const u32", uint32x2x2_t, int32x2_t, "1"] - - ["*const u16", uint16x8x2_t, int16x8_t, "3"] - - ["*const u32", uint32x4x2_t, int32x4_t, "2"] - - ["*const p8", poly8x8x2_t, int8x8_t, "3"] - - ["*const p16", poly16x4x2_t, int16x4_t, "2"] - - ["*const p16", poly16x8x2_t, int16x8_t, "3"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[3]}" - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - FnCall: - - cfg_attr - - - test - - FnCall: - - assert_instr - - - ld2 - - "LANE = 0" - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8, int8x8_t, "3"] - - ["*const i16", int16x4x2_t, i16, int16x4_t, "2"] - - ["*const i32", int32x2x2_t, i32, int32x2_t, "1"] - - ["*const i16", int16x8x2_t, i16, int16x8_t, "3"] - - ["*const i32", int32x4x2_t, i32, int32x4_t, "2"] - - ["*const f32", float32x2x2_t, f32, float32x2_t, "2"] - - ["*const f32", float32x4x2_t, f32, float32x4_t, "2"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[4]}" - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "a: {neon_type[3]}" - - "b: {neon_type[3]}" - - "n: i64" - - "ptr: *const i8" - links: - - link: "llvm.aarch64.neon.ld2lane.v{neon_type[1].lane}{type[2]}.p0i8" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].lane_nox}" - - - "b.0" - - "b.1" - - "LANE as i64" - - "a as _" - - - name: "vld2{neon_type[1].lane_nox}" - doc: Load multiple 2-element structures to two registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - FnCall: - - cfg_attr - - - test - - FnCall: - - assert_instr - - - vld2 - - "LANE = 0" - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-unstable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ["*const i16", int16x8x2_t, i16, int16x8_t, "3"] - - ["*const i32", int32x4x2_t, i32, int32x4_t, "2"] - - ["*const f32", float32x2x2_t, f32, float32x2_t, "1"] - - ["*const f32", float32x4x2_t, f32, float32x4_t, "2"] - compose: - - FnCall: - - "static_assert_uimm_bits!" - - - LANE - - "{type[4]}" - - LLVMLink: - name: "vld2.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "a: {neon_type[3]}" - - "b: {neon_type[3]}" - - "n: i32" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2lane.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2{neon_type[1].lane_nox}" - - - "a as _" - - "b.0" - - "b.1" - - "LANE" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x2_t, i64] - compose: - - LLVMLink: - name: "vld2dup.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2dup.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2{neon_type[1].dup_nox}" - - - "a as *const i8" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - *neon-stable - assert_instr: [ld2r] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x2_t, i64] - compose: - - LLVMLink: - name: "vld2dup.{neon_type[1]}" - arguments: - - "ptr: *const i64" - links: - - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0i64" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].dup_nox}" - - - "a as _" - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [vld2] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8] - - ["*const i16", int16x4x2_t, i16] - - ["*const i32", int32x2x2_t, i32] - - ["*const i8", int8x16x2_t, i8] - - ["*const i16", int16x8x2_t, i16] - - ["*const i32", int32x4x2_t, i32] - - ["*const f32", float32x2x2_t, f32] - - ["*const f32", float32x4x2_t, f32] - compose: - - LLVMLink: - name: "vld2dup.{neon_type[1]}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld2dup.v{neon_type[1].lane}{type[2]}.p0i8" - arch: arm - - FnCall: - - "_vld2{neon_type[1].dup_nox}" - - - "a as *const i8" - - "{neon_type[1].base_byte_size}" - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x8x2_t, int8x8x2_t] - - ["*const u16", uint16x4x2_t, int16x4x2_t] - - ["*const u32", uint32x2x2_t, int32x2x2_t] - - ["*const u8", uint8x16x2_t, int8x16x2_t] - - ["*const u16", uint16x8x2_t, int16x8x2_t] - - ["*const u32", uint32x4x2_t, int32x4x2_t] - - ["*const p8", poly8x8x2_t, int8x8x2_t] - - ["*const p16", poly16x4x2_t, int16x4x2_t] - - ["*const p8", poly8x16x2_t, int8x16x2_t] - - ["*const p16", poly16x8x2_t, int16x8x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - transmute - - - FnCall: - - "vld2{neon_type[2].dup_nox}" - - - FnCall: - - transmute - - - a - - - name: "vld2{neon_type[1].dup_nox}" - doc: Load single 2-element structure and replicate to all lanes of two registers - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - FnCall: - - cfg - - - FnCall: - - not - - - 'target_arch = "arm"' - - *neon-stable - assert_instr: [ld2r] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x2_t, i8] - - ["*const i16", int16x4x2_t, i16] - - ["*const i32", int32x2x2_t, i32] - - ["*const i8", int8x16x2_t, i8] - - ["*const i16", int16x8x2_t, i16] - - ["*const i32", int32x4x2_t, i32] - - ["*const f32", float32x2x2_t, f32] - - ["*const f32", float32x4x2_t, f32] - compose: - - LLVMLink: - name: "vld2dup.{neon_type[1]}" - arguments: - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.ld2r.v{neon_type[1].lane}{type[2]}.p0{type[2]}" - arch: aarch64,arm64ec - - FnCall: - - "_vld2{neon_type[1].dup_nox}" - - - "a as _" - - - name: "vld3{neon_type[1].lane_nox}" - doc: "Load multiple 3-element structures to two registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x3_t, int8x8_t, i8, '3'] - - ['*const i16', int16x8x3_t, int16x8_t, i16, '4'] - - ['*const i32', int32x4x3_t, int32x4_t, i32, '2'] - - ['*const i16', int16x4x3_t, int16x4_t, i16, '2'] - - ['*const i32', int32x2x3_t, int32x2_t, i32, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - LLVMLink: - name: 'ld3lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-stable - - *target-not-arm - assert_instr: [ld3] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x3_t, '*const int8x8_t', i8] - - ['*const i16', int16x4x3_t, '*const int16x4_t', i16] - - ['*const i32', int32x2x3_t, '*const int32x2_t', i32] - - ['*const i8', int8x16x3_t, '*const int8x16_t', i8] - - ['*const i16', int16x8x3_t, '*const int16x8_t', i16] - - ['*const i32', int32x4x3_t, '*const int32x4_t', i32] - - ['*const f32', float32x2x3_t, '*const float32x2_t', f32] - - ['*const f32', float32x4x3_t, '*const float32x4_t', f32] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].nox}' - arguments: - - 'ptr: {type[2]}' - links: - - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-stable - - *target-not-arm - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ['*const i64', int64x1x3_t, '*const int64x1_t', i64] - compose: - - LLVMLink: - name: "vld3{neon_type[1].nox}" - arguments: - - 'ptr: {type[2]}' - links: - - link: 'llvm.aarch64.neon.ld3.v{neon_type[1].lane}{type[3]}.p0v{neon_type[1].lane}{type[3]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].nox}', ['a as _']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vld3] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x3_t, i8] - - ['*const i16', int16x4x3_t, i16] - - ['*const i32', int32x2x3_t, i32] - - ['*const i8', int8x16x3_t, i8] - - ['*const i16', int16x8x3_t, i16] - - ['*const i32', int32x4x3_t, i32] - - ['*const f32', float32x2x3_t, f32] - - ['*const f32', float32x4x3_t, f32] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].nox}', ['a as *const i8', '{neon_type[1].base_byte_size}']] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [nop] - types: - - ['*const i64', int64x1x3_t, i64] - safety: - unsafe: [neon] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].nox}', ['a as *const i8', '{neon_type[1].base_byte_size}']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ['*const f32', float32x4x3_t, float32x4_t, f32, '2'] - - ['*const f32', float32x2x3_t, float32x2_t, f32, '1'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] - - LLVMLink: - name: 'vld3{neon_type[1].lane_nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vld3{neon_type[2].lane_nox}" - doc: "Load multiple 3-element structures to three registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ['*const f32', float32x2x3_t, float32x2_t, f32, '1', '4'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] - - LLVMLink: - name: 'vld3{neon_type[1].lane_nox}' - arguments: - - 'ptr: *const i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] - - - name: "vld3{neon_type[2].lane_nox}" - doc: "Load multiple 3-element structures to two registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x3_t, int8x8_t, i8, '3', '1'] - - ['*const i16', int16x4x3_t, int16x4_t, i16, '2', '2'] - - ['*const i32', int32x2x3_t, int32x2_t, i32, '1', '4'] - - ['*const i16', int16x8x3_t, int16x8_t, i16, '3', '2'] - - ['*const i32', int32x4x3_t, int32x4_t, i32, '2', '4'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] - - LLVMLink: - name: 'vld3{neon_type[1].lane_nox}' - arguments: - - 'ptr: *const i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] - - - name: "vld3{neon_type[2].lane_nox}" - doc: "Load multiple 3-element structures to three registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ['*const f32', float32x4x3_t, float32x4_t, f32, '2', '4'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[4]}']] - - LLVMLink: - name: 'vld3{neon_type[1].lane_nox}' - arguments: - - 'ptr: *const i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', '{type[5]}']] - - - name: "vld3{neon_type[1].lane_nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ['*const u8', uint8x8x3_t, int8x8x3_t, '3'] - - ['*const u16', uint16x4x3_t, int16x4x3_t, '2'] - - ['*const u32', uint32x2x3_t, int32x2x3_t, '1'] - - ['*const p8', poly8x8x3_t, int8x8x3_t, '3'] - - ['*const u16', uint16x8x3_t, int16x8x3_t, '3'] - - ['*const p16', poly16x4x3_t, int16x4x3_t, '2'] - - ['*const p16', poly16x8x3_t, int16x8x3_t, '3'] - - ['*const u32', uint32x4x3_t, int32x4x3_t, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const u8', uint8x8x3_t, int8x8x3_t] - - ['*const u8', uint8x16x3_t, int8x16x3_t] - - ['*const u16', uint16x4x3_t, int16x4x3_t] - - ['*const u32', uint32x2x3_t, int32x2x3_t] - - ['*const u16', uint16x8x3_t, int16x8x3_t] - - ['*const u32', uint32x4x3_t, int32x4x3_t] - - ['*const p8', poly8x8x3_t, int8x8x3_t] - - ['*const p8', poly8x16x3_t, int8x16x3_t] - - ['*const p16', poly16x4x3_t, int16x4x3_t] - - ['*const p16', poly16x8x3_t, int16x8x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].nox}' - - - FnCall: [transmute, [a]] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const u64', uint64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].nox}' - - - FnCall: [transmute, [a]] - - - name: "vld3{neon_type[1].nox}" - doc: Load multiple 3-element structures to three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*target-not-arm, *neon-stable] - assert_instr: [ld3r] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x3_t, i8] - - ["*const i16", int16x4x3_t, i16] - - ["*const i32", int32x2x3_t, i32] - - ["*const i32", int32x4x3_t, i32] - - ["*const i16", int16x8x3_t, i16] - - ["*const i8", int8x16x3_t, i8] - - ["*const i64", int64x1x3_t, i64] - - ["*const f32", float32x4x3_t, f32] - - ["*const f32", float32x2x3_t, f32] - compose: - - LLVMLink: - name: 'ld3r{neon_type[1].dup_nox}' - arguments: - - 'ptr: {type[0]}' - links: - - link: 'llvm.aarch64.neon.ld3r.v{neon_type[1].lane}{type[2]}.p0{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as _']] - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*enable-v7, *target-is-arm, *neon-unstable] - assert_instr: [vld3] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x3_t, i8, '1'] - - ["*const i16", int16x4x3_t, i16, '2'] - - ["*const i32", int32x2x3_t, i32, '4'] - - ["*const i8", int8x16x3_t, i8, '1'] - - ["*const i16", int16x8x3_t, i16, '2'] - - ["*const i32", int32x4x3_t, i32, '4'] - - ["*const f32", float32x4x3_t, f32, '4'] - - ["*const f32", float32x2x3_t, f32, '4'] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].dup_nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3dup.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as *const i8', '{type[3]}']] - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const u8', uint8x8x3_t, int8x8x3_t] - - ['*const u16', uint16x4x3_t, int16x4x3_t] - - ['*const u32', uint32x2x3_t, int32x2x3_t] - - ['*const u8', uint8x16x3_t, int8x16x3_t] - - ['*const u16', uint16x8x3_t, int16x8x3_t] - - ['*const u32', uint32x4x3_t, int32x4x3_t] - - ['*const p8', poly8x8x3_t, int8x8x3_t] - - ['*const p16', poly16x4x3_t, int16x4x3_t] - - ['*const p8', poly8x16x3_t, int8x16x3_t] - - ['*const p16', poly16x8x3_t, int16x8x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].dup_nox}' - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*target-is-arm, *enable-v7, *neon-unstable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x3_t, i64, '8'] - compose: - - LLVMLink: - name: 'vld3{neon_type[1].dup_nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld3dup.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld3{neon_type[1].dup_nox}', ['a as *const i8', '{type[3]}']] - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].dup_nox}' - - - FnCall: - - transmute - - - a - - - name: "vld3{neon_type[1].dup_nox}" - doc: Load single 3-element structure and replicate to all lanes of three registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld3{neon_type[2].dup_nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - *neon-stable - assert_instr: [ld4] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x4_t, i8, '*const int8x8_t'] - - ['*const i32', int32x4x4_t, i32, '*const int32x4_t'] - - ['*const i16', int16x4x4_t, i16, '*const int16x4_t'] - - ['*const i32', int32x2x4_t, i32, '*const int32x2_t'] - - ['*const i8', int8x16x4_t, i8, '*const int8x16_t'] - - ['*const i16', int16x8x4_t, i16, '*const int16x8_t'] - - ['*const f32', float32x2x4_t, f32, '*const float32x2_t'] - - ['*const f32', float32x4x4_t, f32, '*const float32x4_t'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: {type[3]}' - links: - - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: [*target-not-arm, *neon-stable] - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ['*const i64', int64x1x4_t, i64, '*const int64x1_t'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: {type[3]}' - links: - - link: 'llvm.aarch64.neon.ld4.v{neon_type[1].lane}{type[2]}.p0v{neon_type[1].lane}{type[2]}' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].nox}', ['a as _']] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable - static_defs: - - "const LANE: i32" - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x4_t, int8x8_t, i8, '3'] - - ['*const i16', int16x4x4_t, int16x4_t, i16, '2'] - - ['*const i16', int16x8x4_t, int16x8_t, i16, '3'] - - ['*const i32', int32x2x4_t, int32x2_t, i32, '1'] - - ['*const i32', int32x4x4_t, int32x4_t, i32, '2'] - - ['*const f32', float32x2x4_t, float32x2_t, f32, '1'] - - ['*const f32', float32x4x4_t, float32x4_t, f32, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[4]}']] - - LLVMLink: - name: 'ld4lane.{neon_type[2]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i64' - - 'ptr: *const i8' - links: - - link: 'llvm.aarch64.neon.ld4lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vld4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [vld4] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x4_t, i8, '1'] - - ['*const i16', int16x4x4_t, i16, '2'] - - ['*const i32', int32x2x4_t, i32, '4'] - - ['*const i8', int8x16x4_t, i8, '1'] - - ['*const i16', int16x8x4_t, i16, '2'] - - ['*const i32', int32x4x4_t, i32, '4'] - - ['*const f32', float32x4x4_t, f32, '4'] - - ['*const f32', float32x2x4_t, f32, '4'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld4.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld4{neon_type[1].nox}', ['a as *const i8', '{type[3]}']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - ['*const i64', int64x1x4_t, i64, '8'] - compose: - - LLVMLink: - name: 'vld4{neon_type[1].nox}' - arguments: - - 'ptr: *const i8' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld4.v{neon_type[1].lane}{type[2]}.p0i8' - arch: arm - - FnCall: ['_vld4{neon_type[1].nox}', ['a as *const i8', '{type[3]}']] - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const u8', uint8x8x4_t, int8x8x4_t] - - ['*const u16', uint16x4x4_t, int16x4x4_t] - - ['*const u32', uint32x2x4_t, int32x2x4_t] - - ['*const u8', uint8x16x4_t, int8x16x4_t] - - ['*const u16', uint16x8x4_t, int16x8x4_t] - - ['*const u32', uint32x4x4_t, int32x4x4_t] - - ['*const p8', poly8x8x4_t, int8x8x4_t] - - ['*const p16', poly16x4x4_t, int16x4x4_t] - - ['*const p8', poly8x16x4_t, int8x16x4_t] - - ['*const p16', poly16x8x4_t, int16x8x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const u64', uint64x1x4_t, int64x1x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v8 - - FnCall: - - target_feature - - - 'enable = "neon,aes"' - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['*const p64', poly64x1x4_t, int64x1x4_t] - compose: - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].nox}' - - - FnCall: - - transmute - - - a - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *enable-v7 - - *target-is-arm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-unstable - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['*const i8', int8x8x4_t, int8x8_t, i8, '1', '3'] - - ['*const i16', int16x4x4_t, int16x4_t, i16, '2', '2'] - - ['*const i32', int32x2x4_t, int32x2_t, i32, '4', '1'] - - ['*const i16', int16x8x4_t, int16x8_t, i16, '2', '3'] - - ['*const i32', int32x4x4_t, int32x4_t, i32, '4', '2'] - - ['*const f32', float32x2x4_t, float32x2_t, f32, '4', '1'] - - ['*const f32', float32x4x4_t, float32x4_t, f32, '4', '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[5]}']] - - LLVMLink: - name: 'ld4lane.{neon_type[2]}' - arguments: - - 'ptr: *const i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vld4lane.v{neon_type[1].lane}{type[3]}.p0i8' - arch: arm - - FnCall: ['vld4{neon_type[1].lane_nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', LANE, '{type[4]}']] - - - name: "vld4{neon_type[1].lane_nox}" - doc: Load multiple 4-element structures to four registers - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['*const u8', uint8x8x4_t, int8x8x4_t, '3'] - - ['*const u16', uint16x4x4_t, int16x4x4_t, '2'] - - ['*const u32', uint32x2x4_t, int32x2x4_t, '1'] - - ['*const u16', uint16x8x4_t, int16x8x4_t, '3'] - - ['*const u32', uint32x4x4_t, int32x4x4_t, '2'] - - ['*const p8', poly8x8x4_t, int8x8x4_t, '3'] - - ['*const p16', poly16x4x4_t, int16x4x4_t, '2'] - - ['*const p16', poly16x8x4_t, int16x8x4_t, '3'] - compose: - - FnCall: [static_assert_uimm_bits!, ['LANE', '{type[3]}']] - - FnCall: - - transmute - - - FnCall: - - 'vld4{neon_type[2].lane_nox}::' - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - types: - - ['*mut i64', int64x1_t] - - ['*mut u64', uint64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - *neon-v8 - - FnCall: - - target_feature - - - 'enable = "neon,aes"' - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - types: - - ['*mut p64', poly64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - *neon-v8 - - FnCall: - - target_feature - - - 'enable = "neon,aes"' - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - types: - - ['*mut p64', poly64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: "vst1{neon_type[1].lane_nox}" - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ["2"]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - types: - - ['*mut i8', int8x8_t, '3'] - - ['*mut i16', int16x4_t, '2'] - - ['*mut i32', int32x2_t, '1'] - - ['*mut i8', int8x16_t, '4'] - - ['*mut i16', int16x8_t, '3'] - - ['*mut i32', int32x4_t, '2'] - - ['*mut i64', int64x2_t, '1'] - - ['*mut u8', uint8x8_t, '3'] - - ['*mut u16', uint16x4_t, '2'] - - ['*mut u32', uint32x2_t, '1'] - - ['*mut u8', uint8x16_t, '4'] - - ['*mut u16', uint16x8_t, '3'] - - ['*mut u32', uint32x4_t, '2'] - - ['*mut u64', uint64x2_t, '1'] - - ['*mut p8', poly8x8_t, '3'] - - ['*mut p16', poly16x4_t, '2'] - - ['*mut p8', poly8x16_t, '4'] - - ['*mut p16', poly16x8_t, '3'] - - ['*mut f32', float32x2_t, '1'] - - ['*mut f32', float32x4_t, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - Assign: - - "*a" - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - Identifier: [';', Symbol] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [st1] - types: - - [i8, int8x8x2_t, int8x8_t] - - [i16, int16x4x2_t, int16x4_t] - - [i32, int32x2x2_t, int32x2_t] - - [i64, int64x1x2_t, int64x1_t] - - [i8, int8x16x2_t, int8x16_t] - - [i16, int16x8x2_t, int16x8_t] - - [i32, int32x4x2_t, int32x4_t] - - [i64, int64x2x2_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'ptr: *mut {type[0]}' - links: - - link: 'llvm.aarch64.neon.st1x2.v{neon_type[1].lane}{type[0]}.p0{type[0]}' - arch: aarch64,arm64ec - - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'a']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [st1] - types: - - [i8, int8x8x3_t, int8x8_t] - - [i16, int16x4x3_t, int16x4_t] - - [i32, int32x2x3_t, int32x2_t] - - [i64, int64x1x3_t, int64x1_t] - - [i8, int8x16x3_t, int8x16_t] - - [i16, int16x8x3_t, int16x8_t] - - [i32, int32x4x3_t, int32x4_t] - - [i64, int64x2x3_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x3.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'ptr: *mut {type[0]}' - links: - - link: 'llvm.aarch64.neon.st1x3.v{neon_type[1].lane}{type[0]}.p0{type[0]}' - arch: aarch64,arm64ec - - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'b.2', 'a']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [st1] - types: - - [i8, int8x8x4_t, int8x8_t] - - [i16, int16x4x4_t, int16x4_t] - - [i32, int32x2x4_t, int32x2_t] - - [i64, int64x1x4_t, int64x1_t] - - [i8, int8x16x4_t, int8x16_t] - - [i16, int16x8x4_t, int16x8_t] - - [i32, int32x4x4_t, int32x4_t] - - [i64, int64x2x4_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x4.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'ptr: *mut {type[0]}' - links: - - link: 'llvm.aarch64.neon.st1x4.v{neon_type[1].lane}{type[0]}.p0{type[0]}' - arch: aarch64,arm64ec - - FnCall: ['_vst1{neon_type[1].no}', ['b.0', 'b.1', 'b.2', 'b.3', 'a']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [vst1] - types: - - [i8, int8x8x2_t, int8x8_t] - - [i16, int16x4x2_t, int16x4_t] - - [i32, int32x2x2_t, int32x2_t] - - [i64, int64x1x2_t, int64x1_t] - - [i8, int8x16x2_t, int8x16_t] - - [i16, int16x8x2_t, int16x8_t] - - [i32, int32x4x2_t, int32x4_t] - - [i64, int64x2x2_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x2.{neon_type[1]}' - arguments: - - 'ptr: *mut {type[0]}' - - 'a: {type[2]}' - - 'b: {type[2]}' - links: - - link: 'llvm.arm.neon.vst1x2.p0{type[0]}.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *enable-v7 - - *target-is-arm - - *neon-unstable - assert_instr: [vst1] - types: - - [i8, int8x8x3_t, int8x8_t] - - [i16, int16x4x3_t, int16x4_t] - - [i32, int32x2x3_t, int32x2_t] - - [i64, int64x1x3_t, int64x1_t] - - [i8, int8x16x3_t, int8x16_t] - - [i16, int16x8x3_t, int16x8_t] - - [i32, int32x4x3_t, int32x4_t] - - [i64, int64x2x3_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x3.{neon_type[1]}' - arguments: - - 'ptr: *mut {type[0]}' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - links: - - link: 'llvm.arm.neon.vst1x3.p0{type[0]}.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures from one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vst1] - types: - - [i8, int8x8x4_t, int8x8_t] - - [i16, int16x4x4_t, int16x4_t] - - [i32, int32x2x4_t, int32x2_t] - - [i64, int64x1x4_t, int64x1_t] - - [i8, int8x16x4_t, int8x16_t] - - [i16, int16x8x4_t, int16x8_t] - - [i32, int32x4x4_t, int32x4_t] - - [i64, int64x2x4_t, int64x2_t] - compose: - - LLVMLink: - name: 'st1x4.{neon_type[1]}' - arguments: - - 'ptr: *mut {type[0]}' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - links: - - link: 'llvm.arm.neon.vst1x4.p0{type[0]}.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2', 'b.3']] - - - name: 'vst1{neon_type[1].no}' - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - safety: - unsafe: [neon] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vst1] - types: - - [f32, float32x2x4_t, float32x2_t] - - [f32, float32x4x4_t, float32x4_t] - compose: - - LLVMLink: - name: 'st1x4.{neon_type[1]}' - arguments: - - 'ptr: *mut {type[0]}' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - links: - - link: 'llvm.arm.neon.vst1x4.p0{type[0]}.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst1{neon_type[1].no}', ['a', 'b.0', 'b.1', 'b.2', 'b.3']] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v8 - - *neon-aes - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [p64, poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - "vst2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x2_t, int64x1_t] - compose: - - LLVMLink: - name: 'vst2.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst2.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst2{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', '8']] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u64, uint64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - "vst2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x2_t, int64x1_t] - compose: - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [st2] - safety: - unsafe: [neon] - types: - - [i8, int8x8x2_t, int8x8_t] - - [i16, int16x4x2_t, int16x4_t] - - [i32, int32x2x2_t, int32x2_t] - - [i8, int8x16x2_t, int8x16_t] - - [i16, int16x8x2_t, int16x8_t] - - [i32, int32x4x2_t, int32x4_t] - - [f32, float32x2x2_t, float32x2_t] - - [f32, float32x4x2_t, float32x4_t] - compose: - - LLVMLink: - name: 'st2.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].nox}', ['b.0', 'b.1', 'a as _']] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst2]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st2]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u8, uint8x8x2_t, int8x8x2_t] - - [u16, uint16x4x2_t, int16x4x2_t] - - [u32, uint32x2x2_t, int32x2x2_t] - - [u8, uint8x16x2_t, int8x16x2_t] - - [u16, uint16x8x2_t, int16x8x2_t] - - [u32, uint32x4x2_t, int32x4x2_t] - - [p8, poly8x8x2_t, int8x8x2_t] - - [p16, poly16x4x2_t, int16x4x2_t] - - [p8, poly8x16x2_t, int8x16x2_t] - - [p16, poly16x8x2_t, int16x8x2_t] - compose: - - FnCall: - - "vst2{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x2_t, '3', int8x8_t] - - [i16, int16x4x2_t, '2', int16x4_t] - - [i32, int32x2x2_t, '1', int32x2_t] - - [i16, int16x8x2_t, '3', int16x8_t] - - [i32, int32x4x2_t, '2', int32x4_t] - - [f32, float32x2x2_t, '1', float32x2_t] - - [f32, float32x4x2_t, '2', float32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst2.{neon_type[1].lane_nox}' - arguments: - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st2lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst2{neon_type[1].lane_nox}', ['b.0', 'b.1', 'LANE as i64', 'a as _']] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst2, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [u8, uint8x8x2_t, int8x8x2_t, '3'] - - [u16, uint16x4x2_t, int16x4x2_t, '2'] - - [u32, uint32x2x2_t, int32x2x2_t, '1'] - - [u16, uint16x8x2_t, int16x8x2_t, '3'] - - [u32, uint32x4x2_t, int32x4x2_t, '2'] - - [p8, poly8x8x2_t, int8x8x2_t, '3'] - - [p16, poly16x4x2_t, int16x4x2_t, '2'] - - [p16, poly16x8x2_t, int16x8x2_t, '3'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vst2{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst2{neon_type[1].nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vst2] - safety: - unsafe: [neon] - types: - - [i8, int8x8x2_t, int8x8_t, '1'] - - [i16, int16x4x2_t, int16x4_t, '2'] - - [i32, int32x2x2_t, int32x2_t, '4'] - - [i8, int8x16x2_t, int8x16_t, '1'] - - [i16, int16x8x2_t, int16x8_t, '2'] - - [i32, int32x4x2_t, int32x4_t, '4'] - - [f32, float32x2x2_t, float32x2_t, '4'] - - [f32, float32x4x2_t, float32x4_t, '4'] - compose: - - LLVMLink: - name: 'vst2.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst2.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst2{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', "{type[3]}"]] - - - name: "vst2{neon_type[1].lane_nox}" - doc: "Store multiple 2-element structures from two registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst2, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x2_t, '3', int8x8_t, '1'] - - [i16, int16x4x2_t, '2', int16x4_t, '2'] - - [i32, int32x2x2_t, '1', int32x2_t, '4'] - - [i16, int16x8x2_t, '3', int16x8_t, '2'] - - [i32, int32x4x2_t, '2', int32x4_t, '4'] - - [f32, float32x4x2_t, '2', float32x4_t, '4'] - - [f32, float32x2x2_t, '1', float32x2_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst2lane.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst2lane.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst2{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'LANE', "{type[4]}"]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x3_t, int64x1_t] - compose: - - LLVMLink: - name: 'st3.{neon_type[1].nox}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v8 - - *neon-aes - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [p64, poly64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - "vst3{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x3_t, int64x1_t] - compose: - - LLVMLink: - name: 'vst3.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst3.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst3{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', '8']] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u64, uint64x1x3_t, int64x1x3_t] - compose: - - FnCall: - - "vst3{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst3, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [u8, uint8x8x3_t, int8x8x3_t, '3'] - - [u16, uint16x4x3_t, int16x4x3_t, '2'] - - [u32, uint32x2x3_t, int32x2x3_t, '1'] - - [u16, uint16x8x3_t, int16x8x3_t, '3'] - - [u32, uint32x4x3_t, int32x4x3_t, '2'] - - [p8, poly8x8x3_t, int8x8x3_t, '3'] - - [p16, poly16x4x3_t, int16x4x3_t, '2'] - - [p16, poly16x8x3_t, int16x8x3_t, '3'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vst3{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst3]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st3]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u8, uint8x8x3_t, int8x8x3_t] - - [u16, uint16x4x3_t, int16x4x3_t] - - [u32, uint32x2x3_t, int32x2x3_t] - - [u8, uint8x16x3_t, int8x16x3_t] - - [u16, uint16x8x3_t, int16x8x3_t] - - [u32, uint32x4x3_t, int32x4x3_t] - - [p8, poly8x8x3_t, int8x8x3_t] - - [p16, poly16x4x3_t, int16x4x3_t] - - [p8, poly8x16x3_t, int8x16x3_t] - - [p16, poly16x8x3_t, int16x8x3_t] - compose: - - FnCall: - - "vst3{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vst3] - safety: - unsafe: [neon] - types: - - [i8, int8x8x3_t, int8x8_t, '1'] - - [i16, int16x4x3_t, int16x4_t, '2'] - - [i32, int32x2x3_t, int32x2_t, '4'] - - [i8, int8x16x3_t, int8x16_t, '1'] - - [i16, int16x8x3_t, int16x8_t, '2'] - - [i32, int32x4x3_t, int32x4_t, '4'] - - [f32, float32x2x3_t, float32x2_t, '4'] - - [f32, float32x4x3_t, float32x4_t, '4'] - compose: - - LLVMLink: - name: 'vst3.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst3.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst3{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', "{type[3]}"]] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst3, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x3_t, '3', int8x8_t, '1'] - - [i16, int16x4x3_t, '2', int16x4_t, '2'] - - [i32, int32x2x3_t, '1', int32x2_t, '4'] - - [i16, int16x8x3_t, '3', int16x8_t, '2'] - - [i32, int32x4x3_t, '2', int32x4_t, '4'] - - [f32, float32x2x3_t, '1', float32x2_t, '4'] - - [f32, float32x4x3_t, '2', float32x4_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst3lane.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'c: {type[3]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst3lane.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst3{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'LANE', "{type[4]}"]] - - - name: "vst3{neon_type[1].nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*target-not-arm, *neon-stable] - assert_instr: [st3] - safety: - unsafe: [neon] - types: - - [i8, int8x8x3_t, int8x8_t] - - [i16, int16x4x3_t, int16x4_t] - - [i32, int32x2x3_t, int32x2_t] - - [i8, int8x16x3_t, int8x16_t] - - [i16, int16x8x3_t, int16x8_t] - - [i32, int32x4x3_t, int32x4_t] - - [f32, float32x2x3_t, float32x2_t] - - [f32, float32x4x3_t, float32x4_t] - compose: - - LLVMLink: - name: 'vst3.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'a as _']] - - - name: "vst3{neon_type[1].lane_nox}" - doc: "Store multiple 3-element structures from three registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x3_t, '3', int8x8_t] - - [i16, int16x4x3_t, '2', int16x4_t] - - [i32, int32x2x3_t, '1', int32x2_t] - - [i16, int16x8x3_t, '3', int16x8_t] - - [i32, int32x4x3_t, '2', int32x4_t] - - [f32, float32x2x3_t, '1', float32x2_t] - - [f32, float32x4x3_t, '2', float32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst3.{neon_type[1].lane_nox}' - arguments: - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'c: {type[3]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st3lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst3{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'LANE as i64', 'a as _']] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v8 - - *neon-aes - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [p64, poly64x1x4_t, int64x1x4_t] - compose: - - FnCall: - - "vst4{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x4_t, int64x1_t] - compose: - - LLVMLink: - name: 'vst4.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst4.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst4{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', '8']] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - *neon-stable - assert_instr: [nop] - safety: - unsafe: [neon] - types: - - [i64, int64x1x4_t, int64x1_t] - compose: - - LLVMLink: - name: 'vst4.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4.{neon_type[2]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u64, uint64x1x4_t, int64x1x3_t] - compose: - - FnCall: - - "vst4{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst4, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [u8, uint8x8x4_t, int8x8x4_t, '3'] - - [u16, uint16x4x4_t, int16x4x4_t, '2'] - - [u32, uint32x2x4_t, int32x2x4_t, '1'] - - [u16, uint16x8x4_t, int16x8x4_t, '3'] - - [u32, uint32x4x4_t, int32x4x4_t, '2'] - - [p8, poly8x8x4_t, int8x8x4_t, '3'] - - [p16, poly16x4x4_t, int16x4x4_t, '2'] - - [p16, poly16x8x4_t, int16x8x4_t, '3'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vst4{neon_type[2].lane_nox}::" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vst4]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [st4]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [u8, uint8x8x4_t, int8x8x4_t] - - [u16, uint16x4x4_t, int16x4x4_t] - - [u32, uint32x2x4_t, int32x2x4_t] - - [u8, uint8x16x4_t, int8x16x4_t] - - [u16, uint16x8x4_t, int16x8x4_t] - - [u32, uint32x4x4_t, int32x4x4_t] - - [p8, poly8x8x4_t, int8x8x4_t] - - [p16, poly16x4x4_t, int16x4x4_t] - - [p8, poly8x16x4_t, int8x16x4_t] - - [p16, poly16x8x4_t, int16x8x4_t] - compose: - - FnCall: - - "vst4{neon_type[2].nox}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - *neon-unstable - assert_instr: [vst4] - safety: - unsafe: [neon] - types: - - [i8, int8x8x4_t, int8x8_t, '1'] - - [i16, int16x4x4_t, int16x4_t, '2'] - - [i32, int32x2x4_t, int32x2_t, '4'] - - [i8, int8x16x4_t, int8x16_t, '1'] - - [i16, int16x8x4_t, int16x8_t, '2'] - - [i32, int32x4x4_t, int32x4_t, '4'] - - [f32, float32x2x4_t, float32x2_t, '4'] - - [f32, float32x4x4_t, float32x4_t, '4'] - compose: - - LLVMLink: - name: 'vst4.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst4.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst4{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', "{type[3]}"]] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst4, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-unstable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x4_t, '3', int8x8_t, '1'] - - [i16, int16x4x4_t, '2', int16x4_t, '2'] - - [i32, int32x2x4_t, '1', int32x2_t, '4'] - - [i16, int16x8x4_t, '3', int16x8_t, '2'] - - [i32, int32x4x4_t, '2', int32x4_t, '4'] - - [f32, float32x2x4_t, '1', float32x2_t, '4'] - - [f32, float32x4x4_t, '2', float32x4_t, '4'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst4lane.{neon_type[1]}' - arguments: - - 'ptr: *mut i8' - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'c: {type[3]}' - - 'd: {type[3]}' - - 'n: i32' - - 'size: i32' - links: - - link: 'llvm.arm.neon.vst4lane.p0i8.v{neon_type[1].lane}{type[0]}' - arch: arm - - FnCall: ['_vst4{neon_type[1].nox}', ['a as _', 'b.0', 'b.1', 'b.2', 'b.3', 'LANE', "{type[4]}"]] - - - name: "vst4{neon_type[1].nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: [*target-not-arm, *neon-stable] - assert_instr: [st4] - safety: - unsafe: [neon] - types: - - [i8, int8x8x4_t, int8x8_t] - - [i16, int16x4x4_t, int16x4_t] - - [i32, int32x2x4_t, int32x2_t] - - [i8, int8x16x4_t, int8x16_t] - - [i16, int16x8x4_t, int16x8_t] - - [i32, int32x4x4_t, int32x4_t] - - [f32, float32x2x4_t, float32x2_t] - - [f32, float32x4x4_t, float32x4_t] - compose: - - LLVMLink: - name: 'vst4.{neon_type[1]}' - arguments: - - 'a: {type[2]}' - - 'b: {type[2]}' - - 'c: {type[2]}' - - 'd: {type[2]}' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'a as _']] - - - name: "vst4{neon_type[1].lane_nox}" - doc: "Store multiple 4-element structures from four registers" - arguments: ["a: *mut {type[0]}", "b: {neon_type[1]}"] - attr: - - *target-not-arm - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]] - - *neon-stable - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [i8, int8x8x4_t, '3', int8x8_t] - - [i16, int16x4x4_t, '2', int16x4_t] - - [i32, int32x2x4_t, '1', int32x2_t] - - [i16, int16x8x4_t, '3', int16x8_t] - - [i32, int32x4x4_t, '2', int32x4_t] - - [f32, float32x2x4_t, '1', float32x2_t] - - [f32, float32x4x4_t, '2', float32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - LLVMLink: - name: 'vst4.{neon_type[1].lane_nox}' - arguments: - - 'a: {type[3]}' - - 'b: {type[3]}' - - 'c: {type[3]}' - - 'd: {type[3]}' - - 'n: i64' - - 'ptr: *mut i8' - links: - - link: 'llvm.aarch64.neon.st4lane.v{neon_type[1].lane}{type[0]}.p0i8' - arch: aarch64,arm64ec - - FnCall: ['_vst4{neon_type[1].lane_nox}', ['b.0', 'b.1', 'b.2', 'b.3', 'LANE as i64', 'a as _']] - - - name: "vusdot{neon_type[0].no}" - doc: "Dot product vector form with unsigned and signed integers" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-i8mm - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vusdot]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usdot]]}]] - - *neon-unstable-i8mm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int32x2_t, uint8x8_t, int8x8_t] - - [int32x4_t, uint8x16_t, int8x16_t] - compose: - - LLVMLink: - name: "usdot.{neon_type[0]}" - links: - - link: "llvm.aarch64.neon.usdot.v{neon_type[0].lane}i32.v{neon_type[1].lane}i8" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.usdot.v{neon_type[0].lane}i32.v{neon_type[1].lane}i8" - arch: arm - - - name: "vusdot{type[0]}" - doc: "Dot product index form with unsigned and signed integers" - arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}", "c: int8x8_t"] - return_type: "{neon_type[1]}" - attr: - - *neon-i8mm - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vusdot, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usdot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-i8mm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - ['_lane_s32', int32x2_t, uint8x8_t, '[LANE as u32, LANE as u32]'] - - ['q_lane_s32', int32x4_t, uint8x16_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Let: - - c - - int32x2_t - - FnCall: [transmute, [c]] - - Let: - - c - - "{type[1]}" - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - FnCall: ["vusdot{neon_type[1].no}", [a, b, {FnCall: [transmute, [c]]}]] - - - name: "vsudot{neon_type[0].lane_nox}" - doc: "Dot product index form with signed and unsigned integers" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-i8mm - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsudot, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sudot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - *neon-unstable-i8mm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [int32x2_t, int8x8_t, uint8x8_t, '[LANE as u32, LANE as u32]', uint32x2_t] - - [int32x4_t, int8x16_t, uint8x8_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]', uint32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Let: - - c - - uint32x2_t - - FnCall: [transmute, [c]] - - Let: - - c - - "{type[4]}" - - FnCall: [simd_shuffle!, [c, c, "{type[3]}"]] - - FnCall: ["vusdot{neon_type[0].no}", [a, {FnCall: [transmute, [c]]}, b]] - - - name: "vmul{neon_type[1].no}" - doc: Multiply - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmul{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['.i8', int8x8_t] - - ['.i8', int8x16_t] - - ['.i16', int16x4_t] - - ['.i16', int16x8_t] - - ['.i32', int32x2_t] - - ['.i32', int32x4_t] - - ['.i8', uint8x8_t] - - ['.i8', uint8x16_t] - - ['.i16', uint16x4_t] - - ['.i16', uint16x8_t] - - ['.i32', uint32x2_t] - - ['.i32', uint32x4_t] - compose: - - FnCall: [simd_mul, [a, b]] - - - name: "vmul{neon_type[1].no}" - doc: Multiply - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmul.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmul]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [f32, float32x2_t] - - [f32, float32x4_t] - compose: - - FnCall: [simd_mul, [a, b]] - - - name: "vmul{neon_type[0].lane_nox}" - doc: Multiply - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmul, 'LANE = 1']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] - - FnCall: - - simd_mul - - - a - - FnCall: ["simd_shuffle!", [b, b, "{type[3]}"]] - - - name: "vmul{neon_type[0].laneq_nox}" - doc: Multiply - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmul, 'LANE = 1']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [mul, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - *neon-stable-not-arm - - *neon-unstable-is-arm - static_defs: ["const LANE: i32"] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: ["static_assert_uimm_bits!", [LANE, "{type[2]}"]] - - FnCall: - - simd_mul - - - a - - FnCall: ["simd_shuffle!", [b, b, "{type[3]}"]] - - - name: "vmull{neon_type[1].no}" - doc: Signed multiply long - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smull]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["s8", int8x8_t, int16x8_t] - - ["s16", int16x4_t, int32x4_t] - - ["s32", int32x2_t, int64x2_t] - compose: - - LLVMLink: - name: "smull.{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.smull.{neon_type[1]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vmulls.{neon_type[1]}" - arch: arm - - - name: "vmull{neon_type[1].no}" - doc: "Unsigned multiply long" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umull]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["u8", uint8x8_t, uint16x8_t] - - ["u16", uint16x4_t, uint32x4_t] - - ["u32", uint32x2_t, uint64x2_t] - compose: - - LLVMLink: - name: "smull.{neon_type[1]}" - links: - - link: "llvm.aarch64.neon.umull.{neon_type[1]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vmullu.{neon_type[1]}" - arch: arm - - - name: "vmull{neon_type[1].no}" - doc: "Polynomial multiply long" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vmull.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [pmull]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ["p8", poly8x8_t, poly16x8_t, int8x8_t] - compose: - - LLVMLink: - name: "pmull.{neon_type[1].no}" - links: - - link: "llvm.aarch64.neon.pmull.{neon_type[3]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vmullp.{neon_type[3]}" - arch: arm - - - name: "vmull_n{neon_type[0].no}" - doc: Vector long multiply with scalar - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vmull"]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smull]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", int32x4_t] - - [int32x2_t, "i32", int64x2_t] - compose: - - FnCall: - - "vmull{neon_type[0].no}" - - - a - - FnCall: - - "vdup_n{neon_type[0].no}" - - - b - - - name: "vmull_n{neon_type[0].no}" - doc: Vector long multiply with scalar - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vmull"]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umull]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint16x4_t, "u16", uint32x4_t] - - [uint32x2_t, "u32", uint64x2_t] - compose: - - FnCall: - - "vmull{neon_type[0].no}" - - - a - - FnCall: - - "vdup_n{neon_type[0].no}" - - - b - - - name: "vfma{neon_type.no}" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - vfma - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - fmla - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "fma.{neon_type}" - links: - - link: "llvm.fma.{neon_type}" - arch: aarch64 - - link: "llvm.fma.{neon_type}" - arch: arm - - FnCall: ["_vfma{neon_type.no}", [b, c, a]] - - - name: "vfma{neon_type[0].N}" - doc: Floating-point fused Multiply-Add to accumulator(vector) - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - vfma - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - fmla - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - [float32x2_t, f32] - - [float32x4_t, f32] - compose: - - FnCall: - - "vfma{neon_type[0].no}" - - - a - - b - - FnCall: - - "vdup{neon_type[0].N}_vfp4" - - - c - - - name: "vsub{neon_type[1].no}" - doc: "Subtract" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vsub{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['.i8', int8x8_t] - - ['.i8', int8x16_t] - - ['.i16', int16x4_t] - - ['.i16', int16x8_t] - - ['.i32', int32x2_t] - - ['.i32', int32x4_t] - - ['.i8', uint8x8_t] - - ['.i8', uint8x16_t] - - ['.i16', uint16x4_t] - - ['.i16', uint16x8_t] - - ['.i32', uint32x2_t] - - ['.i32', uint32x4_t] - - ['.i64', int64x1_t] - - ['.i64', int64x2_t] - - ['.i64', uint64x1_t] - - ['.i64', uint64x2_t] - compose: - - FnCall: [simd_sub, [a, b]] - - - name: "vsub{neon_type[1].no}" - doc: "Subtract" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vsub.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fsub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['f32', float32x2_t] - - ['f32', float32x4_t] - compose: - - FnCall: [simd_sub, [a, b]] - - - name: "vadd{neon_type.no}" - doc: Bitwise exclusive OR - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - target_feature - - - 'enable = "v7"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - nop - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - nop - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - poly8x8_t - - poly16x4_t - - poly8x16_t - - poly16x8_t - - poly64x1_t - - poly64x2_t - compose: - - FnCall: - - simd_xor - - - a - - b - - - name: "vaddq_{type}" - doc: Bitwise exclusive OR - arguments: ["a: {type}", "b: {type}"] - return_type: "{type}" - attr: - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - target_feature - - - 'enable = "v7"' - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - 'target_arch = "arm"' - - FnCall: - - assert_instr - - - nop - - FnCall: - - cfg_attr - - - FnCall: - - all - - - test - - FnCall: - - any - - - 'target_arch = "aarch64"' - - 'target_arch = "arm64ec"' - - FnCall: - - assert_instr - - - nop - - FnCall: - - cfg_attr - - - FnCall: - - not - - - 'target_arch = "arm"' - - FnCall: - - stable - - - 'feature = "neon_intrinsics"' - - 'since = "1.59.0"' - - FnCall: - - cfg_attr - - - 'target_arch = "arm"' - - FnCall: - - unstable - - - 'feature = "stdarch_arm_neon_intrinsics"' - - 'issue = "111800"' - safety: - unsafe: [neon] - types: - - p128 - compose: - - Xor: - - a - - b - - - name: "vsubhn{neon_type[0].noq}" - doc: Subtract returning high narrow - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vsubhn"]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [subhn]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'i16x8', 'i16x8::new(8, 8, 8, 8, 8, 8, 8, 8)'] - - [int32x4_t, int16x4_t, 'i32x4', 'i32x4::new(16, 16, 16, 16)'] - - [int64x2_t, int32x2_t, 'i64x2', 'i64x2::new(32, 32)'] - - [uint16x8_t, uint8x8_t, 'u16x8', 'u16x8::new(8, 8, 8, 8, 8, 8, 8, 8)'] - - [uint32x4_t, uint16x4_t, 'u32x4', 'u32x4::new(16, 16, 16, 16)'] - - [uint64x2_t, uint32x2_t, 'u64x2', 'u64x2::new(32, 32)'] - compose: - - Let: [c, "{type[2]}", "{type[3]}"] - - FnCall: - - simd_cast - - - FnCall: - - simd_shr - - - FnCall: [simd_sub, [a, b]] - - FnCall: [transmute, [c]] - - - name: "vsubhn_high{neon_type[1].noq}" - doc: Subtract returning high narrow - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ["vsubhn"]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [subhn2]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, int8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [int16x4_t, int32x4_t, int16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [int32x2_t, int64x2_t, int32x4_t, '[0, 1, 2, 3]'] - - [uint8x8_t, uint16x8_t, uint8x16_t, '[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]'] - - [uint16x4_t, uint32x4_t, uint16x8_t, '[0, 1, 2, 3, 4, 5, 6, 7]'] - - [uint32x2_t, uint64x2_t, uint32x4_t, '[0, 1, 2, 3]'] - compose: - - Let: - - d - - "{neon_type[0]}" - - FnCall: ["vsubhn{neon_type[1].noq}", [b, c]] - - FnCall: [simd_shuffle!, [a, d, "{type[3]}"]] - - - name: "vhsub{neon_type[1].no}" - doc: "Signed halving subtract" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vhsub.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uhsub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['u8', uint8x8_t] - - ['u8', uint8x16_t] - - ['u16', uint16x4_t] - - ['u16', uint16x8_t] - - ['u32', uint32x2_t] - - ['u32', uint32x4_t] - compose: - - LLVMLink: - name: "uhsub.{neon_type[1].no}" - links: - - link: "llvm.aarch64.neon.uhsub.{neon_type[1]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vhsubu.{neon_type[1]}" - arch: arm - - - name: "vhsub{neon_type[1].no}" - doc: "Signed halving subtract" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vhsub.{type[0]}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [shsub]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - ['s8', int8x8_t] - - ['s8', int8x16_t] - - ['s16', int16x4_t] - - ['s16', int16x8_t] - - ['s32', int32x2_t] - - ['s32', int32x4_t] - compose: - - LLVMLink: - name: "shsub.{neon_type[1].no}" - links: - - link: "llvm.aarch64.neon.shsub.{neon_type[1]}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vhsubs.{neon_type[1]}" - arch: arm - - - name: "vsubw{neon_type[1].noq}" - doc: Signed Subtract Wide - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubw]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ssubw]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t] - - [int32x4_t, int16x4_t] - - [int64x2_t, int32x2_t] - compose: - - FnCall: - - simd_sub - - - a - - FnCall: [simd_cast, [b]] - - - name: "vsubw{neon_type[1].noq}" - doc: Unsigned Subtract Wide - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubw]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usubw]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t] - - [uint32x4_t, uint16x4_t] - - [uint64x2_t, uint32x2_t] - compose: - - FnCall: - - simd_sub - - - a - - FnCall: [simd_cast, [b]] - - - name: "vsubl{neon_type[0].noq}" - doc: "Signed Subtract Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubl]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ssubl]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t] - - [int16x4_t, int32x4_t] - - [int32x2_t, int64x2_t] - compose: - - Let: - - c - - "{neon_type[1]}" - - FnCall: [simd_cast, [a]] - - Let: - - d - - "{neon_type[1]}" - - FnCall: [simd_cast, [b]] - - FnCall: [simd_sub, [c, d]] - - - name: "vsubl{neon_type[0].noq}" - doc: "Unsigned Subtract Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsubl]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [usubl]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint16x8_t] - - [uint16x4_t, uint32x4_t] - - [uint32x2_t, uint64x2_t] - compose: - - Let: - - c - - "{neon_type[1]}" - - FnCall: [simd_cast, [a]] - - Let: - - d - - "{neon_type[1]}" - - FnCall: [simd_cast, [b]] - - FnCall: [simd_sub, [c, d]] - - - name: "vdot{neon_type[0].no}" - doc: Dot product arithmetic (vector) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v8 - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsdot]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sdot]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int32x2_t, int8x8_t] - - [int32x4_t, int8x16_t] - compose: - - LLVMLink: - name: "sdot.{neon_type[0]}.{neon_type[1]}" - links: - - link: "llvm.arm.neon.sdot.{neon_type[0]}.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.sdot.{neon_type[0]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vdot{neon_type[0].no}" - doc: Dot product arithmetic (vector) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - *neon-v8 - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vudot]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [udot]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint32x2_t, uint8x8_t] - - [uint32x4_t, uint8x16_t] - compose: - - LLVMLink: - name: "udot.{neon_type[0]}.{neon_type[1]}" - links: - - link: "llvm.arm.neon.udot.{neon_type[0]}.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.udot.{neon_type[0]}.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vdot{neon_type[0].lane_nox}" - doc: Dot product arithmetic (indexed) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - static_defs: ["const LANE: i32"] - attr: - - *neon-v8 - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vsdot, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sdot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int32x2_t, int8x8_t, int8x8_t, int32x2_t, '[LANE as u32, LANE as u32]'] - - [int32x4_t, int8x16_t, int8x8_t, int32x2_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Let: - - c - - "{neon_type[3]}" - - FnCall: [transmute, [c]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] - - FnCall: - - "vdot{neon_type[0].no}" - - - a - - b - - FnCall: [transmute, [c]] - - - name: "vdot{neon_type[0].lane_nox}" - doc: Dot product arithmetic (indexed) - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[0]}" - static_defs: ["const LANE: i32"] - attr: - - *neon-v8 - - FnCall: [target_feature, ['enable = "neon,dotprod"']] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vudot, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [udot, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [unstable, ['feature = "stdarch_neon_dotprod"', 'issue = "117224"']]}]] - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint32x2_t, uint8x8_t, uint8x8_t, uint32x2_t, '[LANE as u32, LANE as u32]'] - - [uint32x4_t, uint8x16_t, uint8x8_t, uint32x2_t, '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - Let: - - c - - "{neon_type[3]}" - - FnCall: [transmute, [c]] - - Let: - - c - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [c, c, '{type[4]}']] - - FnCall: - - "vdot{neon_type[0].no}" - - - a - - b - - FnCall: [transmute, [c]] - - - name: "vmax{neon_type.no}" - doc: Maximum (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smax]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "smax.{neon_type}" - links: - - link: "llvm.arm.neon.vmaxs.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.smax.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmax{neon_type.no}" - doc: Maximum (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umax]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "smax.{neon_type}" - links: - - link: "llvm.arm.neon.vmaxu.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.umax.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmax{neon_type.no}" - doc: Maximum (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmax]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmax]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "smax.{neon_type}" - links: - - link: "llvm.arm.neon.vmaxs.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.fmax.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmaxnm{neon_type.no}" - doc: Floating-point Maximum Number (vector) - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmaxnm]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmaxnm]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "fmaxnm.{neon_type}" - links: - - link: "llvm.arm.neon.vmaxnm.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.fmaxnm.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmin{neon_type.no}" - doc: "Minimum (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [smin]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "smin.{neon_type}" - links: - - link: "llvm.arm.neon.vmins.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.smin.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmin{neon_type.no}" - doc: "Minimum (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [umin]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "umin.{neon_type}" - links: - - link: "llvm.arm.neon.vminu.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.umin.{neon_type}" - arch: aarch64,arm64ec - - - name: "vmin{neon_type.no}" - doc: "Minimum (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vmin]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmin]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "fmin.{neon_type}" - links: - - link: "llvm.arm.neon.vmins.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.fmin.{neon_type}" - arch: aarch64,arm64ec - - - name: "vminnm{neon_type.no}" - doc: "Floating-point Minimum Number (vector)" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [target_feature, ['enable = "fp-armv8,v8"']]}]] - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vminnm]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fminnm]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "fminnm.{neon_type}" - links: - - link: "llvm.arm.neon.vminnm.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.fminnm.{neon_type}" - arch: aarch64,arm64ec - - - name: "vpadd{neon_type.no}" - doc: Floating-point add pairwise - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vpadd]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [faddp]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - float32x2_t - compose: - - LLVMLink: - name: "faddp.{neon_type}" - links: - - link: "llvm.arm.neon.vpadd.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.faddp.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqdmull{neon_type[0].noq}" - doc: "Signed saturating doubling multiply long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, int32x4_t] - - [int32x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vqdmull{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vqdmull.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.sqdmull.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqdmull_n{neon_type[0].no}" - doc: "Vector saturating doubling long multiply with scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", int32x4_t] - - [int32x2_t, "i32", int64x2_t] - compose: - - FnCall: ["vqdmull{neon_type[0].noq}", [a, {FnCall: ["vdup_n{neon_type[0].noq}", [b]]}]] - - - name: "vqdmull_lane_s16" - doc: "Vector saturating doubling long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, int32x4_t, '[N as u32, N as u32, N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - FnCall: [vqdmull_s16, [a, b]] - - - name: "vqdmull_lane_s32" - doc: "Vector saturating doubling long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmull, 'N = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmull, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, int32x2_t, int64x2_t, '[N as u32, N as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '1']] - - Let: [b, "{neon_type[0]}", {FnCall: [simd_shuffle!, [b, b, "{type[3]}"]]}] - - FnCall: [vqdmull_s32, [a, b]] - - - name: "vqdmlal{neon_type[1].noq}" - doc: "Signed saturating doubling multiply-add long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] - - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] - compose: - - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull{neon_type[2].noq}", [b, c]]}]] - - - name: "vqdmlal_n{neon_type[1].noq}" - doc: "Vector widening saturating doubling multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, "i16", int32x4_t] - - [int64x2_t, int32x2_t, "i32", int64x2_t] - compose: - - FnCall: ["vqadd{neon_type[0].no}", [a, {FnCall: ["vqdmull_n{neon_type[1].noq}", [b, c]]}]] - - - name: "vqdmlal_lane_s16" - doc: "Vector widening saturating doubling multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal, N = 2]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - FnCall: [vqaddq_s32, [a, {FnCall: ["vqdmull_lane_s16::", [b, c]]}]] - - - name: "vqdmlal_lane_s32" - doc: "Vector widening saturating doubling multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlal, N = 1]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlal, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '1']] - - FnCall: [vqaddq_s64, [a, {FnCall: ["vqdmull_lane_s32::", [b, c]]}]] - - - name: "vqdmlsl{neon_type[1].noq}" - doc: "Signed saturating doubling multiply-subtract long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] - - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] - compose: - - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{neon_type[1].noq}", [b, c]]}]] - - - name: "vqdmlsl{type[4]}" - doc: "Vector widening saturating doubling multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {type[2]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, "i16", int32x4_t, '_n_s16'] - - [int64x2_t, int32x2_t, "i32", int64x2_t, '_n_s32'] - compose: - - FnCall: ["vqsub{neon_type[0].no}", [a, {FnCall: ["vqdmull{type[4]}", [b, c]]}]] - - - name: "vqdmlsl_lane_s16" - doc: "Vector widening saturating doubling multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl, N = 2]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x4_t, int16x4_t, int16x4_t, int32x4_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '2']] - - FnCall: [vqsubq_s32, [a, {FnCall: ["vqdmull_lane_s16::", [b, c]]}]] - - - name: "vqdmlsl_lane_s32" - doc: "Vector widening saturating doubling multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmlsl, N = 1]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmlsl, 'N = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int64x2_t, int32x2_t, int32x2_t, int64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, '1']] - - FnCall: [vqsubq_s64, [a, {FnCall: ["vqdmull_lane_s32::", [b, c]]}]] - - - name: "vqdmulh{neon_type[0].no}" - doc: "Signed saturating doubling multiply returning high half" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, int16x4_t] - - [int16x8_t, int16x8_t, int16x8_t] - - [int32x2_t, int32x2_t, int32x2_t] - - [int32x4_t, int32x4_t, int32x4_t] - compose: - - LLVMLink: - name: "vqdmulh{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vqdmulh.{neon_type[0]}" - arch: arm - - link: "llvm.aarch64.neon.sqdmulh.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vqdmulh{type[3]}" - doc: "Vector saturating doubling multiply high with scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", int16x4_t, '_n_s16'] - - [int32x2_t, "i32", int32x2_t, '_n_s32'] - - [int16x8_t, "i16", int16x8_t, 'q_n_s16'] - - [int32x4_t, "i32", int32x4_t, 'q_n_s32'] - compose: - - Let: [b, "{neon_type[0]}", {FnCall: ["vdup{type[3]}", [b]]}] - - FnCall: ["vqdmulh{neon_type[0].no}", [a, b]] - - - name: "vqmovn{neon_type[0].noq}" - doc: "Signed saturating extract narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqxtn]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t] - - [int32x4_t, int16x4_t] - - [int64x2_t, int32x2_t] - compose: - - LLVMLink: - name: "vqmovn{neon_type[0].noq}" - links: - - link: "llvm.arm.neon.vqmovns.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.sqxtn.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqmovun{neon_type[0].noq}" - doc: "Signed saturating extract unsigned narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovun]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqxtun]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, uint8x8_t] - - [int32x4_t, uint16x4_t] - - [int64x2_t, uint32x2_t] - compose: - - LLVMLink: - name: "vqmovun{neon_type[0].noq}" - links: - - link: "llvm.arm.neon.vqmovnsu.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.sqxtun.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqrdmulh{neon_type[0].no}" - doc: "Signed saturating rounding doubling multiply returning high half" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, int16x4_t] - - [int16x8_t, int16x8_t, int16x8_t] - - [int32x2_t, int32x2_t, int32x2_t] - - [int32x4_t, int32x4_t, int32x4_t] - compose: - - LLVMLink: - name: "vqrdmulh{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vqrdmulh.{neon_type[0]}" - arch: arm - - link: "llvm.aarch64.neon.sqrdmulh.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vqrshl{neon_type.no}" - doc: "Signed saturating rounding shift left" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "vqrshl{neon_type}" - links: - - link: "llvm.arm.neon.vqrshifts.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.sqrshl.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqrshl{neon_type[0].no}" - doc: "Unsigned signed saturating rounding shift left" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [uint16x4_t, int16x4_t] - - [uint16x8_t, int16x8_t] - - [uint32x2_t, int32x2_t] - - [uint32x4_t, int32x4_t] - - [uint64x1_t, int64x1_t] - - [uint64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vqrshl{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vqrshiftu.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.uqrshl.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqrshrn_n{neon_type[0].noq}" - doc: "Signed saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int16x8_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int16x8_t([-N as i64, -N as i64]) }'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqrshrn{neon_type[0].noq}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqrshiftns.{neon_type[1]}" - arch: arm - - FnCall: ["_vqrshrn_n{neon_type[0].noq}", [a, '{type[3]}']] - - - name: "vqrshrn_n{neon_type[0].noq}" - doc: "Signed saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqrshrn{neon_type[0].no}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.sqrshrn.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqrshrn_n{neon_type[0].noq}", [a, N]] - - - name: "vqrshrun_n{neon_type[0].noq}" - doc: "Signed saturating rounded shift right unsigned narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] - - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] - - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - LLVMLink: - name: "vqrshrun_n{neon_type[0].noq}" - arguments: - - 'a: {neon_type[0]}' - - 'n: {neon_type[0]}' - links: - - link: "llvm.arm.neon.vqrshiftnsu.{neon_type[1]}" - arch: arm - - FnCall: - - "_vqrshrun_n{neon_type[0].noq}" - - - a - - "{type[3]}" - - - name: "vqrshrun_n{neon_type[0].noq}" - doc: "Signed saturating rounded shift right unsigned narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqrshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] - - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] - - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqrshrun_n{neon_type[0].noq}" - arguments: - - 'a: {neon_type[0]}' - - 'n: i32' - links: - - link: "llvm.aarch64.neon.sqrshrun.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqrshrun_n{neon_type[0].noq}", [a, N]] - - - name: "vqshl{neon_type.no}" - doc: "Signed saturating shift left" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "vqshl{neon_type}" - links: - - link: "llvm.arm.neon.vqshifts.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.sqshl.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqshl{neon_type[0].N}" - doc: "Signed saturating shift left" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqshl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, '3'] - - [int8x16_t, '3'] - - [int16x4_t, '4'] - - [int16x8_t, '4'] - - [int32x2_t, '5'] - - [int32x4_t, '5'] - - [int64x1_t, '6'] - - [int64x2_t, '6'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] - - FnCall: - - "vqshl{neon_type[0].no}" - - - a - - FnCall: ["vdup{neon_type[0].N}", ['N as _']] - - - name: "vqshl{neon_type[0].no}" - doc: "Unsigned saturating shift left" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [uint16x4_t, int16x4_t] - - [uint16x8_t, int16x8_t] - - [uint32x2_t, int32x2_t] - - [uint32x4_t, int32x4_t] - - [uint64x1_t, int64x1_t] - - [uint64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vqshl{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vqshiftu.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.uqshl.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vqshl{neon_type[0].N}" - doc: "Unsigned saturating shift left" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqshl, N = 2]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqshl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, '3', int8x8_t] - - [uint8x16_t, '3', int8x16_t] - - [uint16x4_t, '4', int16x4_t] - - [uint16x8_t, '4', int16x8_t] - - [uint32x2_t, '5', int32x2_t] - - [uint32x4_t, '5', int32x4_t] - - [uint64x1_t, '6', int64x1_t] - - [uint64x2_t, '6', int64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] - - FnCall: - - "vqshl{neon_type[0].no}" - - - a - - FnCall: ["vdup{neon_type[2].N}", ['N as _']] - - - name: "vqshrn_n{neon_type[0].noq}" - doc: "Signed saturating shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrn{neon_type[0].no}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqshiftns.{neon_type[1]}" - arch: arm - - FnCall: ["_vqshrn_n{neon_type[0].noq}", [a, "{type[3]}"]] - - - name: "vqshrn_n{neon_type[0].noq}" - doc: "Signed saturating shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrn_n{neon_type[0].noq}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.sqshrn.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqshrn_n{neon_type[0].noq}", [a, N]] - - - name: "vqshrn_n_{neon_type[0]}" - doc: "Unsigned saturating shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { uint16x8_t([-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16]) }'] - - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }'] - - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { uint64x2_t([-N as u64, -N as u64]) }'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrn_n_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqshiftnu.{neon_type[1]}" - arch: arm - - FnCall: ["_vqshrn_n_{neon_type[0]}", ["a.as_signed()", "{type[3]}"]] - - - name: "vqshrn_n_{neon_type[0]}" - doc: "Unsigned saturating shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] - - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] - - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrn{neon_type[1].no}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.uqshrn.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqshrn_n_{neon_type[0]}", ["a.as_signed()", N]] - - - name: "vqshrun_n_{neon_type[0]}" - doc: "Signed saturating shift right unsigned narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] - - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] - - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrun_n_{neon_type[1]}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqshiftnsu.{neon_type[1]}" - arch: arm - - FnCall: ["_vqshrun_n_{neon_type[1]}", [a, "{type[3]}"]] - - - name: "vqshrun_n_{neon_type[0]}" - doc: "Signed saturating shift right unsigned narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshrun, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, uint8x8_t, 'N >= 1 && N <= 8'] - - [int32x4_t, uint16x4_t, 'N >= 1 && N <= 16'] - - [int64x2_t, uint32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vqshrun_n_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.sqshrun.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqshrun_n_{neon_type[0]}", [a, N]] - - - name: "vrsqrts{neon_type.no}" - doc: "Floating-point reciprocal square root step" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrts]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frsqrts]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vrsqrts{neon_type.no}" - links: - - link: "llvm.arm.neon.vrsqrts.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.frsqrts.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrecpe{neon_type.no}" - doc: "Reciprocal estimate." - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecpe]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frecpe]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vrecpe{neon_type.no}" - links: - - link: "llvm.arm.neon.vrecpe.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.frecpe.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrecps{neon_type.no}" - doc: "Floating-point reciprocal step" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecps]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frecps]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vrecps{neon_type.no}" - links: - - link: "llvm.arm.neon.vrecps.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.frecps.{neon_type}" - arch: aarch64,arm64ec - - - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" - doc: Vector reinterpret cast operation - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-aes - - *neon-v8 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [poly64x1_t, int32x2_t] - - [poly64x1_t, uint32x2_t] - - [poly64x2_t, int32x4_t] - - [poly64x2_t, uint32x4_t] - - [p128, int64x2_t] - - [p128, uint64x2_t] - - [p128, poly64x2_t] - - [poly8x16_t, p128] - - [p128, int8x16_t] - - [p128, uint8x16_t] - - [p128, poly8x16_t] - - [int32x2_t, poly64x1_t] - - [uint32x2_t, poly64x1_t] - - [int32x4_t, poly64x2_t] - - [uint32x4_t, poly64x2_t] - - [int64x2_t, p128] - - [uint64x2_t, p128] - - [poly64x2_t, p128] - - [poly64x1_t, int16x4_t] - - [poly64x1_t, uint16x4_t] - - [poly64x1_t, poly16x4_t] - - [poly64x2_t, int16x8_t] - - [poly64x2_t, uint16x8_t] - - [poly64x2_t, poly16x8_t] - - [p128, int32x4_t] - - [p128, uint32x4_t] - - [poly16x4_t, poly64x1_t] - - [int16x4_t, poly64x1_t] - - [uint16x4_t, poly64x1_t] - - [poly16x8_t, poly64x2_t] - - [int16x8_t, poly64x2_t] - - [uint16x8_t, poly64x2_t] - - [int32x4_t, p128] - - [uint32x4_t, p128] - - [poly64x1_t, int8x8_t] - - [poly64x1_t, uint8x8_t] - - [poly64x1_t, poly8x8_t] - - [poly64x2_t, int8x16_t] - - [poly64x2_t, uint8x16_t] - - [poly64x2_t, poly8x16_t] - - [p128, int16x8_t] - - [p128, uint16x8_t] - - [p128, poly16x8_t] - - [poly8x8_t, poly64x1_t] - - [int8x8_t, poly64x1_t] - - [uint8x8_t, poly64x1_t] - - [poly8x16_t, poly64x2_t] - - [int8x16_t, poly64x2_t] - - [uint8x16_t, poly64x2_t] - - [int16x8_t, p128] - - [uint16x8_t, p128] - - [poly16x8_t, p128] - - [int8x16_t, p128] - - [uint8x16_t, p128] - compose: - - FnCall: [transmute, [a]] - - - name: "vreinterpret{neon_type[1].no}{neon_type[0].noq}" - doc: Vector reinterpret cast operation - arguments: ["a: {type[0]}"] - return_type: "{type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [poly8x8_t, int8x8_t] - - [poly16x4_t, int16x4_t] - - [uint16x4_t, int16x4_t] - - [uint32x2_t, int32x2_t] - - [uint64x1_t, int64x1_t] - - [uint8x16_t, int8x16_t] - - [poly8x16_t, int8x16_t] - - [poly16x8_t, int16x8_t] - - [uint16x8_t, int16x8_t] - - [uint32x4_t, int32x4_t] - - [uint64x2_t, int64x2_t] - - [poly8x8_t, uint8x8_t] - - [int8x8_t, uint8x8_t] - - [poly16x4_t, uint16x4_t] - - [int16x4_t, uint16x4_t] - - [int32x2_t, uint32x2_t] - - [int64x1_t, uint64x1_t] - - [poly8x16_t, uint8x16_t] - - [int8x16_t, uint8x16_t] - - [poly16x8_t, uint16x8_t] - - [int16x8_t, uint16x8_t] - - [int32x4_t, uint32x4_t] - - [int64x2_t, uint64x2_t] - - [int8x8_t, poly8x8_t] - - [uint8x8_t, poly8x8_t] - - [int16x4_t, poly16x4_t] - - [uint16x4_t, poly16x4_t] - - [int8x16_t, poly8x16_t] - - [uint8x16_t, poly8x16_t] - - [int16x8_t, poly16x8_t] - - [uint16x8_t, poly16x8_t] - - [int16x4_t, int8x8_t] - - [uint16x4_t, int8x8_t] - - [poly16x4_t, int8x8_t] - - [int32x2_t, int16x4_t] - - [uint32x2_t, int16x4_t] - - [int64x1_t, int32x2_t] - - [uint64x1_t, int32x2_t] - - [int16x8_t, int8x16_t] - - [uint16x8_t, int8x16_t] - - [poly16x8_t, int8x16_t] - - [int32x4_t, int16x8_t] - - [uint32x4_t, int16x8_t] - - [int64x2_t, int32x4_t] - - [uint64x2_t, int32x4_t] - - [poly16x4_t, uint8x8_t] - - [int16x4_t, uint8x8_t] - - [uint16x4_t, uint8x8_t] - - [int32x2_t, uint16x4_t] - - [uint32x2_t, uint16x4_t] - - [int64x1_t, uint32x2_t] - - [uint64x1_t, uint32x2_t] - - [poly16x8_t, uint8x16_t] - - [int16x8_t, uint8x16_t] - - [uint16x8_t, uint8x16_t] - - [int32x4_t, uint16x8_t] - - [uint32x4_t, uint16x8_t] - - [int64x2_t, uint32x4_t] - - [uint64x2_t, uint32x4_t] - - [poly16x4_t, poly8x8_t] - - [int16x4_t, poly8x8_t] - - [uint16x4_t, poly8x8_t] - - [int32x2_t, poly16x4_t] - - [uint32x2_t, poly16x4_t] - - [poly16x8_t, poly8x16_t] - - [int16x8_t, poly8x16_t] - - [uint16x8_t, poly8x16_t] - - [int32x4_t, poly16x8_t] - - [uint32x4_t, poly16x8_t] - - [poly8x8_t, int16x4_t] - - [int8x8_t, int16x4_t] - - [uint8x8_t, int16x4_t] - - [poly16x4_t, int32x2_t] - - [int16x4_t, int32x2_t] - - [uint16x4_t, int32x2_t] - - [int32x2_t, int64x1_t] - - [uint32x2_t, int64x1_t] - - [poly8x16_t, int16x8_t] - - [int8x16_t, int16x8_t] - - [uint8x16_t, int16x8_t] - - [poly16x8_t, int32x4_t] - - [int16x8_t, int32x4_t] - - [uint16x8_t, int32x4_t] - - [int32x4_t, int64x2_t] - - [uint32x4_t, int64x2_t] - - [poly8x8_t, uint16x4_t] - - [int8x8_t, uint16x4_t] - - [uint8x8_t, uint16x4_t] - - [poly16x4_t, uint32x2_t] - - [int16x4_t, uint32x2_t] - - [uint16x4_t, uint32x2_t] - - [int32x2_t, uint64x1_t] - - [uint32x2_t, uint64x1_t] - - [poly8x16_t, uint16x8_t] - - [int8x16_t, uint16x8_t] - - [uint8x16_t, uint16x8_t] - - [poly16x8_t, uint32x4_t] - - [int16x8_t, uint32x4_t] - - [uint16x8_t, uint32x4_t] - - [int32x4_t, uint64x2_t] - - [uint32x4_t, uint64x2_t] - - [poly8x8_t, poly16x4_t] - - [int8x8_t, poly16x4_t] - - [uint8x8_t, poly16x4_t] - - [poly8x16_t, poly16x8_t] - - [int8x16_t, poly16x8_t] - - [uint8x16_t, poly16x8_t] - - [int32x2_t, int8x8_t] - - [uint32x2_t, int8x8_t] - - [int64x1_t, int16x4_t] - - [uint64x1_t, int16x4_t] - - [int32x4_t, int8x16_t] - - [uint32x4_t, int8x16_t] - - [int64x2_t, int16x8_t] - - [uint64x2_t, int16x8_t] - - [int32x2_t, uint8x8_t] - - [uint32x2_t, uint8x8_t] - - [int64x1_t, uint16x4_t] - - [uint64x1_t, uint16x4_t] - - [int32x4_t, uint8x16_t] - - [uint32x4_t, uint8x16_t] - - [int64x2_t, uint16x8_t] - - [uint64x2_t, uint16x8_t] - - [int32x2_t, poly8x8_t] - - [uint32x2_t, poly8x8_t] - - [int64x1_t, poly16x4_t] - - [uint64x1_t, poly16x4_t] - - [int32x4_t, poly8x16_t] - - [uint32x4_t, poly8x16_t] - - [int64x2_t, poly16x8_t] - - [uint64x2_t, poly16x8_t] - - [poly8x8_t, int32x2_t] - - [int8x8_t, int32x2_t] - - [uint8x8_t, int32x2_t] - - [poly16x4_t, int64x1_t] - - [int16x4_t, int64x1_t] - - [uint16x4_t, int64x1_t] - - [poly8x16_t, int32x4_t] - - [int8x16_t, int32x4_t] - - [uint8x16_t, int32x4_t] - - [poly16x8_t, int64x2_t] - - [int16x8_t, int64x2_t] - - [uint16x8_t, int64x2_t] - - [poly8x8_t, uint32x2_t] - - [int8x8_t, uint32x2_t] - - [uint8x8_t, uint32x2_t] - - [poly16x4_t, uint64x1_t] - - [int16x4_t, uint64x1_t] - - [uint16x4_t, uint64x1_t] - - [poly8x16_t, uint32x4_t] - - [int8x16_t, uint32x4_t] - - [uint8x16_t, uint32x4_t] - - [poly16x8_t, uint64x2_t] - - [int16x8_t, uint64x2_t] - - [uint16x8_t, uint64x2_t] - - [int64x1_t, int8x8_t] - - [uint64x1_t, int8x8_t] - - [int64x1_t, uint8x8_t] - - [uint64x1_t, uint8x8_t] - - [int64x1_t, poly8x8_t] - - [uint64x1_t, poly8x8_t] - - [int64x2_t, int8x16_t] - - [uint64x2_t, int8x16_t] - - [int64x2_t, uint8x16_t] - - [uint64x2_t, uint8x16_t] - - [int64x2_t, poly8x16_t] - - [uint64x2_t, poly8x16_t] - - [poly8x8_t, int64x1_t] - - [int8x8_t, int64x1_t] - - [uint8x8_t, int64x1_t] - - [poly8x8_t, uint64x1_t] - - [int8x8_t, uint64x1_t] - - [uint8x8_t, uint64x1_t] - - [poly8x16_t, int64x2_t] - - [int8x16_t, int64x2_t] - - [uint8x16_t, int64x2_t] - - [poly8x16_t, uint64x2_t] - - [int8x16_t, uint64x2_t] - - [uint8x16_t, uint64x2_t] - - [float32x2_t, int8x8_t] - - [float32x2_t, int16x4_t] - - [float32x2_t, int32x2_t] - - [float32x2_t, int64x1_t] - - [float32x4_t, int8x16_t] - - [float32x4_t, int16x8_t] - - [float32x4_t, int32x4_t] - - [float32x4_t, int64x2_t] - - [float32x2_t, uint8x8_t] - - [float32x2_t, uint16x4_t] - - [float32x2_t, uint32x2_t] - - [float32x2_t, uint64x1_t] - - [float32x4_t, uint8x16_t] - - [float32x4_t, uint16x8_t] - - [float32x4_t, uint32x4_t] - - [float32x4_t, uint64x2_t] - - [float32x2_t, poly8x8_t] - - [float32x2_t, poly16x4_t] - - [float32x4_t, poly8x16_t] - - [float32x4_t, poly16x8_t] - - [float32x4_t, p128] - - [int8x8_t, float32x2_t] - - [int16x4_t, float32x2_t] - - [int32x2_t, float32x2_t] - - [int64x1_t, float32x2_t] - - [int8x16_t, float32x4_t] - - [int16x8_t, float32x4_t] - - [int32x4_t, float32x4_t] - - [int64x2_t, float32x4_t] - - [uint8x8_t, float32x2_t] - - [uint16x4_t, float32x2_t] - - [uint32x2_t, float32x2_t] - - [uint64x1_t, float32x2_t] - - [uint8x16_t, float32x4_t] - - [uint16x8_t, float32x4_t] - - [uint32x4_t, float32x4_t] - - [uint64x2_t, float32x4_t] - - [poly8x8_t, float32x2_t] - - [poly16x4_t, float32x2_t] - - [poly8x16_t, float32x4_t] - - [poly16x8_t, float32x4_t] - - [p128, float32x4_t] - compose: - - FnCall: [transmute, [a]] - - - name: "vrshl{neon_type.no}" - doc: "Signed rounding shift left" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "vrshl{neon_type.no}" - links: - - link: "llvm.arm.neon.vrshifts.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.srshl.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrshl{neon_type[0].no}" - doc: "Unsigned rounding shift left" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [uint16x4_t, int16x4_t] - - [uint16x8_t, int16x8_t] - - [uint32x2_t, int32x2_t] - - [uint32x4_t, int32x4_t] - - [uint64x1_t, int64x1_t] - - [uint64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vrshl{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vrshiftu.{neon_type[0]}" - arch: arm - - link: "llvm.aarch64.neon.urshl.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vrshr{neon_type[0].N}" - doc: "Signed rounding shift right" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshr, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, 'N >= 1 && N <= 8'] - - [int8x16_t, 'N >= 1 && N <= 8'] - - [int16x4_t, 'N >= 1 && N <= 16'] - - [int16x8_t, 'N >= 1 && N <= 16'] - - [int32x2_t, 'N >= 1 && N <= 32'] - - [int32x4_t, 'N >= 1 && N <= 32'] - - [int64x1_t, 'N >= 1 && N <= 64'] - - [int64x2_t, 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - FnCall: - - "vrshl{neon_type[0].no}" - - - a - - FnCall: ["vdup{neon_type[0].N}", ['-N as _']] - - - name: "vrshr{neon_type[0].N}" - doc: "Unsigned rounding shift right" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshr, N = 2]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t, 'N >= 1 && N <= 8'] - - [uint8x16_t, int8x16_t, 'N >= 1 && N <= 8'] - - [uint16x4_t, int16x4_t, 'N >= 1 && N <= 16'] - - [uint16x8_t, int16x8_t, 'N >= 1 && N <= 16'] - - [uint32x2_t, int32x2_t, 'N >= 1 && N <= 32'] - - [uint32x4_t, int32x4_t, 'N >= 1 && N <= 32'] - - [uint64x1_t, int64x1_t, 'N >= 1 && N <= 64'] - - [uint64x2_t, int64x2_t, 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - FnCall: - - "vrshl{neon_type[0].no}" - - - a - - FnCall: ["vdup{neon_type[1].N}", ['-N as _']] - - - name: "vrshrn_n_{neon_type[0]}" - doc: "Rounding shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8', 'const { int16x8_t([-N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16, -N as i16]) }'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16', 'const { int32x4_t([-N as i32, -N as i32, -N as i32, -N as i32]) }'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32', 'const { int64x2_t([-N as i64, -N as i64]) }'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vrshrn_n_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vrshiftn.{neon_type[1]}" - arch: arm - - FnCall: ["_vrshrn_n_{neon_type[0]}", [a, "{type[3]}"]] - - - name: "vrshrn_n_{neon_type[0]}" - doc: "Rounding shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [rshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, 'N >= 1 && N <= 8'] - - [int32x4_t, int16x4_t, 'N >= 1 && N <= 16'] - - [int64x2_t, int32x2_t, 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - LLVMLink: - name: "vrshrn_n_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.rshrn.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vrshrn_n_{neon_type[0]}", [a, N]] - - - name: "vrshrn_n_{neon_type[0]}" - doc: "Rounding shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrshrn, N = 2]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, 'N >= 1 && N <= 8', s16] - - [uint32x4_t, uint16x4_t, 'N >= 1 && N <= 16', s32] - - [uint64x2_t, uint32x2_t, 'N >= 1 && N <= 32', s64] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - FnCall: - - transmute - - - FnCall: - - "vrshrn_n_{type[3]}::" - - - FnCall: [transmute, [a]] - - - name: "vrsra{neon_type[0].N}" - doc: "Signed rounding shift right and accumulate" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsra, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [srsra, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, 'N >= 1 && N <= 8'] - - [int8x16_t, 'N >= 1 && N <= 8'] - - [int16x4_t, 'N >= 1 && N <= 16'] - - [int16x8_t, 'N >= 1 && N <= 16'] - - [int32x2_t, 'N >= 1 && N <= 32'] - - [int32x4_t, 'N >= 1 && N <= 32'] - - [int64x1_t, 'N >= 1 && N <= 64'] - - [int64x2_t, 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - FnCall: - - simd_add - - - a - - FnCall: ["vrshr{neon_type[0].N}::", [b]] - - - name: "vrsubhn_{neon_type[0]}" - doc: "Rounding subtract returning high narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsubhn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rsubhn]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int16x8_t, int8x8_t] - - [int32x4_t, int32x4_t, int16x4_t] - - [int64x2_t, int64x2_t, int32x2_t] - compose: - - LLVMLink: - name: "vrsubhn_{neon_type[0]}" - links: - - link: "llvm.arm.neon.vrsubhn.{neon_type[2]}" - arch: arm - - link: "llvm.aarch64.neon.rsubhn.{neon_type[2]}" - arch: aarch64,arm64ec - - - name: "vrsubhn_{neon_type[0]}" - doc: "Rounding subtract returning high narrow" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsubhn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [rsubhn]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint16x8_t, uint8x8_t, s16] - - [uint32x4_t, uint32x4_t, uint16x4_t, s32] - - [uint64x2_t, uint64x2_t, uint32x2_t, s64] - compose: - - FnCall: - - transmute - - - FnCall: - - "vrsubhn_{type[3]}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vcreate_{neon_type[1]}" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["u64", int8x8_t] - - ["u64", int16x4_t] - - ["u64", int32x2_t] - - ["u64", int64x1_t] - - ["u64", uint8x8_t] - - ["u64", uint16x4_t] - - ["u64", uint32x2_t] - - ["u64", uint64x1_t] - - ["u64", poly8x8_t] - - ["u64", poly16x4_t] - - ["u64", float32x2_t] - compose: - - FnCall: [transmute, [a]] - - - name: "vcreate_p64" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["u64", poly64x1_t] - compose: - - FnCall: [transmute, [a]] - - - name: "vset{neon_type[1].lane_nox}" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, LANE = 0]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i8", int8x8_t, '3'] - - ["i16", int16x4_t, '2'] - - ["i32", int32x2_t, '1'] - - ["u8", uint8x8_t, '3'] - - ["u16", uint16x4_t, '2'] - - ["u32", uint32x2_t, '1'] - - ["p8", poly8x8_t, '3'] - - ["p16", poly16x4_t, '2'] - - ["i8", int8x16_t, '4'] - - ["i16", int16x8_t, '3'] - - ["i32", int32x4_t, '2'] - - ["i64", int64x2_t, '1'] - - ["u8", uint8x16_t, '4'] - - ["u16", uint16x8_t, '3'] - - ["u32", uint32x4_t, '2'] - - ["u64", uint64x2_t, '1'] - - ["p8", poly8x16_t, '4'] - - ["p16", poly16x8_t, '3'] - - ["f32", float32x2_t, '1'] - - ["f32", float32x4_t, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vset_lane_{neon_type[0]}" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["i64", int64x1_t, int64x1_t] - - ["u64", uint64x1_t, uint64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vset_lane_{neon_type[0]}" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["p64", poly64x1_t, poly64x1_t] - compose: - - FnCall: [static_assert!, ['LANE == 0']] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vsetq_lane_p64" - doc: "Insert vector element from another vector element" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - ["p64", poly64x2_t, poly64x2_t] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '1']] - - FnCall: [simd_insert!, [b, 'LANE as u32', a]] - - - name: "vshl{neon_type.no}" - doc: "Signed Shift left" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - - int64x1_t - - int64x2_t - compose: - - LLVMLink: - name: "vshl{neon_type.no}" - links: - - link: "llvm.arm.neon.vshifts.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.sshl.{neon_type}" - arch: aarch64,arm64ec - - - name: "vshl{neon_type[0].no}" - doc: "Unsigned Shift left" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, int8x8_t] - - [uint8x16_t, int8x16_t] - - [uint16x4_t, int16x4_t] - - [uint16x8_t, int16x8_t] - - [uint32x2_t, int32x2_t] - - [uint32x4_t, int32x4_t] - - [uint64x1_t, int64x1_t] - - [uint64x2_t, int64x2_t] - compose: - - LLVMLink: - name: "vshl{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vshiftu.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.ushl.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vshll_n_s8" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s8"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, int16x8_t, 'N >= 0 && N <= 8'] - compose: - - FnCall: [static_assert!, ["{type[2]}"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_s16, ['N as _']] - - - name: "vshll_n_s16" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s16"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, int32x4_t] - compose: - - FnCall: [static_assert!, ["N >= 0 && N <= 16"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_s32, ['N as _']] - - - name: "vshll_n_s32" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.s32"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int32x2_t, int64x2_t] - compose: - - FnCall: [static_assert!, ["N >= 0 && N <= 32"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_s64, ['N as _']] - - - name: "vshll_n_u8" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u8"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint16x8_t] - compose: - - FnCall: [static_assert!, ["N >= 0 && N <= 8"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_u16, ['N as _']] - - - name: "vshll_n_u16" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u16"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x4_t, uint32x4_t] - compose: - - FnCall: [static_assert!, ["N >= 0 && N <= 16"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_u32, ['N as _']] - - - name: "vshll_n_u32" - doc: "Signed shift left long" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshll.u32"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushll, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint32x2_t, uint64x2_t] - compose: - - FnCall: [static_assert!, ["N >= 0 && N <= 32"]] - - FnCall: - - simd_shl - - - FnCall: [simd_cast, [a]] - - FnCall: [vdupq_n_u64, ['N as _']] - - - name: "vshr{neon_type[0].N}" - doc: "Shift right" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshr.{neon_type[0]}"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sshr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { 7 }', 'else { N };'] - - [int8x16_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { 7 }', 'else { N };'] - - [int16x4_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { 15 }', 'else { N };'] - - [int16x8_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { 15 }', 'else { N };'] - - [int32x2_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { 31 }', 'else { N };'] - - [int32x4_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { 31 }', 'else { N };'] - - [int64x1_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { 63 }', 'else { N };'] - - [int64x2_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { 63 }', 'else { N };'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - Identifier: ["{type[2]}{type[3]}", Symbol] - - FnCall: - - simd_shr - - - a - - FnCall: ["vdup{neon_type[0].N}", ['n as _']] - - - name: "vshr{neon_type[0].N}" - doc: "Shift right" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshr.{neon_type[0]}"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ushr, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { return vdup_n_u8(0); }', 'else { N };'] - - [uint8x16_t, 'N >= 1 && N <= 8', 'let n: i32 = if N == 8 { return vdupq_n_u8(0); }', 'else { N };'] - - [uint16x4_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { return vdup_n_u16(0); }', 'else { N };'] - - [uint16x8_t, 'N >= 1 && N <= 16', 'let n: i32 = if N == 16 { return vdupq_n_u16(0); }', 'else { N };'] - - [uint32x2_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { return vdup_n_u32(0); }', 'else { N };'] - - [uint32x4_t, 'N >= 1 && N <= 32', 'let n: i32 = if N == 32 { return vdupq_n_u32(0); }', 'else { N };'] - - [uint64x1_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { return vdup_n_u64(0); }', 'else { N };'] - - [uint64x2_t, 'N >= 1 && N <= 64', 'let n: i32 = if N == 64 { return vdupq_n_u64(0); }', 'else { N };'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - Identifier: ['{type[2]}{type[3]}', Symbol] - - FnCall: - - simd_shr - - - a - - FnCall: ["vdup{neon_type[0].N}", ['n as _']] - - - name: "vshrn_n_{neon_type[0]}" - doc: "Shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vshrn{type[2]}"', 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [shrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, '.i16', 'N >= 1 && N <= 8'] - - [uint16x8_t, uint8x8_t, '.i16', 'N >= 1 && N <= 8'] - - [int32x4_t, int16x4_t, '.i32', 'N >= 1 && N <= 16'] - - [uint32x4_t, uint16x4_t, '.i32', 'N >= 1 && N <= 16'] - - [int64x2_t, int32x2_t, '.i64', 'N >= 1 && N <= 32'] - - [uint64x2_t, uint32x2_t, '.i64', 'N >= 1 && N <= 32'] - compose: - - FnCall: [static_assert!, ["{type[3]}"]] - - FnCall: - - simd_cast - - - FnCall: - - simd_shr - - - a - - FnCall: ["vdupq_n_{neon_type[0]}", ['N as _']] - - - name: "vsra{neon_type[0].N}" - doc: "Signed shift right and accumulate" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vsra, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ssra, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, 'N >= 1 && N <= 8'] - - [int8x16_t, 'N >= 1 && N <= 8'] - - [int16x4_t, 'N >= 1 && N <= 16'] - - [int16x8_t, 'N >= 1 && N <= 16'] - - [int32x2_t, 'N >= 1 && N <= 32'] - - [int32x4_t, 'N >= 1 && N <= 32'] - - [int64x1_t, 'N >= 1 && N <= 64'] - - [int64x2_t, 'N >= 1 && N <= 64'] - compose: - - FnCall: [static_assert!, ["{type[1]}"]] - - FnCall: - - simd_add - - - a - - FnCall: ["vshr{neon_type[0].N}::", [b]] - - - name: "vtrn{neon_type[0].no}" - doc: "Transpose elements" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtrn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [trn]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int8x8_t, int8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [int16x4_t, int16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - - [int8x16_t, int8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [int16x8_t, int16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [int32x4_t, int32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - - [uint8x8_t, uint8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [uint16x4_t, uint16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - - [uint8x16_t, uint8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [uint16x8_t, uint16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [uint32x4_t, uint32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - - [poly8x8_t, poly8x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [poly16x4_t, poly16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - - [poly8x16_t, poly8x16x2_t, '[0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30]', '[1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31]'] - - [poly16x8_t, poly16x8x2_t, '[0, 8, 2, 10, 4, 12, 6, 14]', '[1, 9, 3, 11, 5, 13, 7, 15]'] - - [float32x4_t, float32x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]'] - compose: - - Let: - - a1 - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [a, b, "{type[2]}"]] - - Let: - - b1 - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - Identifier: ['(a1, b1)', Symbol] - - - name: "vtrn{neon_type[0].no}" - doc: "Transpose elements" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtrn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [zip]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] - - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] - - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] - compose: - - Let: - - a1 - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [a, b, "{type[2]}"]] - - Let: - - b1 - - "{neon_type[0]}" - - FnCall: [simd_shuffle!, [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - Identifier: ['(a1, b1)', Symbol] - - - name: "vzip{neon_type[0].no}" - doc: Zip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vorr]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x16_t, int8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [int16x8_t, int16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [int32x4_t, int32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - - [uint8x16_t, uint8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [uint16x8_t, uint16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [uint32x4_t, uint32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - - [poly8x16_t, poly8x16x2_t, '[0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]', '[8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]'] - - [poly16x8_t, poly16x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [float32x4_t, float32x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - compose: - - Let: - - a0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] - - Let: - - b0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - '(a0, b0)' - - - name: "vzip{neon_type[0].no}" - doc: Zip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vtrn]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] - - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] - - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] - compose: - - Let: - - a0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] - - Let: - - b0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - '(a0, b0)' - - - name: "vzip{neon_type[0].no}" - doc: Zip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vzip]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, int8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [int16x4_t, int16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - - [uint8x8_t, uint8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [uint16x4_t, uint16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - - [poly8x8_t, poly8x8x2_t, '[0, 8, 1, 9, 2, 10, 3, 11]', '[4, 12, 5, 13, 6, 14, 7, 15]'] - - [poly16x4_t, poly16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]'] - compose: - - Let: - - a0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] - - Let: - - b0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - '(a0, b0)' - - - name: "vuzp{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vuzp]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uzp]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [int8x8_t, int8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [int16x4_t, int16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - - [int8x16_t, int8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [int16x8_t, int16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [int32x4_t, int32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - - [uint8x8_t, uint8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [uint16x4_t, uint16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - - [uint8x16_t, uint8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [uint16x8_t, uint16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [uint32x4_t, uint32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - - [poly8x8_t, poly8x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [poly16x4_t, poly16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - - [poly8x16_t, poly8x16x2_t, '[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]', '[1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]'] - - [poly16x8_t, poly16x8x2_t, '[0, 2, 4, 6, 8, 10, 12, 14]', '[1, 3, 5, 7, 9, 11, 13, 15]'] - - [float32x4_t, float32x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]'] - compose: - - Let: - - a0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] - - Let: - - b0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - '(a0, b0)' - - - name: "vuzp{neon_type[0].no}" - doc: Unzip vectors - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vtrn]]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2x2_t, '[0, 2]', '[1, 3]'] - - [int32x2_t, int32x2x2_t, '[0, 2]', '[1, 3]'] - - [uint32x2_t, uint32x2x2_t, '[0, 2]', '[1, 3]'] - compose: - - Let: - - a0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[2]}"]] - - Let: - - b0 - - "{neon_type[0]}" - - FnCall: ["simd_shuffle!", [a, b, "{type[3]}"]] - - FnCall: - - transmute - - - '(a0, b0)' - - - name: "vabal_{neon_type[1]}" - doc: "Unsigned Absolute difference and Accumulate Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabal.{type[2]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uabal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, "u8"] - - [uint32x4_t, uint16x4_t, "u16"] - - [uint64x2_t, uint32x2_t, "u32"] - compose: - - Let: [d, "{neon_type[1]}", {FnCall: ["vabd_{type[2]}", [b, c]]}] - - FnCall: [simd_add, [a, {FnCall: [simd_cast, [d]]}]] - - - name: "vabal_{neon_type[1]}" - doc: "Signed Absolute difference and Accumulate Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}", "c: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabal.{neon_type[1]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sabal]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x8_t, int8x8_t, uint8x8_t] - - [int32x4_t, int16x4_t, uint16x4_t] - - [int64x2_t, int32x2_t, uint32x2_t] - compose: - - Let: [d, "{type[1]}", {FnCall: ["vabd_{neon_type[1]}", [b, c]]}] - - Let: [e, "{type[2]}", {FnCall: ["simd_cast", [d]]}] - - FnCall: [simd_add, [a, {FnCall: [simd_cast, [e]]}]] - - - name: "vqabs{neon_type.no}" - doc: Signed saturating Absolute value - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - *neon-v7 - - FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vqabs.{neon_type}"']]}]] - - FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [sqabs]]}]] - - *neon-stable-not-arm - - *neon-unstable-is-arm - safety: - unsafe: [neon] - types: - - int8x8_t - - int8x16_t - - int16x4_t - - int16x8_t - - int32x2_t - - int32x4_t - compose: - - LLVMLink: - name: "sqabs.{neon_type}" - links: - - link: "llvm.aarch64.neon.sqabs.{neon_type}" - arch: aarch64,arm64ec - - link: "llvm.arm.neon.vqabs.{neon_type}" - arch: arm - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vst1]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*mut u8", uint8x8x2_t, int8x8x2_t] - - ["*mut u16", uint16x4x2_t, int16x4x2_t] - - ["*mut u32", uint32x2x2_t, int32x2x2_t] - - ["*mut u64", uint64x1x2_t, int64x1x2_t] - - ["*mut u8", uint8x16x2_t, int8x16x2_t] - - ["*mut u16", uint16x8x2_t, int16x8x2_t] - - ["*mut u32", uint32x4x2_t, int32x4x2_t] - - ["*mut u64", uint64x2x2_t, int64x2x2_t] - - ["*mut u8", uint8x8x3_t, int8x8x3_t] - - ["*mut u16", uint16x4x3_t, int16x4x3_t] - - ["*mut u32", uint32x2x3_t, int32x2x3_t] - - ["*mut u64", uint64x1x3_t, int64x1x3_t] - - ["*mut u8", uint8x16x3_t, int8x16x3_t] - - ["*mut u16", uint16x8x3_t, int16x8x3_t] - - ["*mut u32", uint32x4x3_t, int32x4x3_t] - - ["*mut u64", uint64x2x3_t, int64x2x3_t] - - ["*mut u8", uint8x8x4_t, int8x8x4_t] - - ["*mut u16", uint16x4x4_t, int16x4x4_t] - - ["*mut u32", uint32x2x4_t, int32x2x4_t] - - ["*mut u64", uint64x1x4_t, int64x1x4_t] - - ["*mut u8", uint8x16x4_t, int8x16x4_t] - - ["*mut u16", uint16x8x4_t, int16x8x4_t] - - ["*mut u32", uint32x4x4_t, int32x4x4_t] - - ["*mut u64", uint64x2x4_t, int64x2x4_t] - - ["*mut p8", poly8x8x2_t, int8x8x2_t] - - ["*mut p8", poly8x8x3_t, int8x8x3_t] - - ["*mut p8", poly8x8x4_t, int8x8x4_t] - - ["*mut p8", poly8x16x2_t, int8x16x2_t] - - ["*mut p8", poly8x16x3_t, int8x16x3_t] - - ["*mut p8", poly8x16x4_t, int8x16x4_t] - - ["*mut p16", poly16x4x2_t, int16x4x2_t] - - ["*mut p16", poly16x4x3_t, int16x4x3_t] - - ["*mut p16", poly16x4x4_t, int16x4x4_t] - - ["*mut p16", poly16x8x2_t, int16x8x2_t] - - ["*mut p16", poly16x8x3_t, int16x8x3_t] - - ["*mut p16", poly16x8x4_t, int16x8x4_t] - compose: - - FnCall: - - "vst1{neon_type[2].no}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vst1]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*mut p64", poly64x1x2_t, int64x1x2_t] - compose: - - FnCall: - - "vst1{neon_type[2].no}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*mut p64", poly64x1x3_t, int64x1x3_t] - - ["*mut p64", poly64x1x4_t, int64x1x4_t] - - ["*mut p64", poly64x2x2_t, int64x2x2_t] - - ["*mut p64", poly64x2x3_t, int64x2x3_t] - - ["*mut p64", poly64x2x4_t, int64x2x4_t] - compose: - - FnCall: - - "vst1{neon_type[2].no}" - - - FnCall: [transmute, [a]] - - FnCall: [transmute, [b]] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - safety: - unsafe: [neon] - types: - - ['*mut f32', float32x2x2_t, float32x2_t] - - ['*mut f32', float32x4x2_t, float32x4_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "ptr: {type[0]}" - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - links: - - link: "llvm.arm.neon.vst1x{neon_type[1].tuple}.p0f32.{neon_type[2]}" - arch: arm - - FnCall: ["_vst1{neon_type[1].no}", ['a', 'b.0', 'b.1']] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - safety: - unsafe: [neon] - types: - - ['*mut f32', float32x2x3_t, float32x2_t] - - ['*mut f32', float32x4x3_t, float32x4_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "ptr: {type[0]}" - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "c: {neon_type[2]}" - links: - - link: "llvm.arm.neon.vst1x{neon_type[1].tuple}.p0f32.{neon_type[2]}" - arch: arm - - FnCall: ["_vst1{neon_type[1].no}", ['a', 'b.0', 'b.1', 'b.2']] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*mut f32", float32x2x2_t, float32x2_t] - - ["*mut f32", float32x4x2_t, float32x4_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x2.{neon_type[2]}.p0f32" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', a]] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*mut f32", float32x2x3_t, float32x2_t] - - ["*mut f32", float32x4x3_t, float32x4_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "c: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x3.{neon_type[2]}.p0f32" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', a]] - - - name: "vst1{neon_type[1].no}" - doc: "Store multiple single-element structures to one, two, three, or four registers" - arguments: ["a: {type[0]}", "b: {neon_type[1]}"] - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*mut f32", float32x2x4_t, float32x2_t] - - ["*mut f32", float32x4x4_t, float32x4_t] - compose: - - LLVMLink: - name: "vst1{neon_type[1].no}" - arguments: - - "a: {neon_type[2]}" - - "b: {neon_type[2]}" - - "c: {neon_type[2]}" - - "d: {neon_type[2]}" - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.st1x4.{neon_type[2]}.p0f32" - arch: aarch64,arm64ec - - FnCall: ["_vst1{neon_type[1].no}", ['b.0', 'b.1', 'b.2', 'b.3', a]] - - - name: "vfms{neon_type.no}" - doc: "Floating-point fused multiply-subtract from accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vfms]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - Let: [b, "{neon_type}", {FnCall: [simd_neg, [b]]}] - - FnCall: ["vfma{neon_type.no}", [a, b, c]] - - - name: "vmul{neon_type[0].no}" - doc: "Polynomial multiply" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [pmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [poly8x8_t, int8x8_t] - - [poly8x16_t, int8x16_t] - compose: - - LLVMLink: - name: "vmul{neon_type[0].no}" - links: - - link: "llvm.arm.neon.vmulp.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.pmul.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vmls{neon_type.no}" - doc: "Floating-point multiply-subtract from accumulator" - arguments: ["a: {neon_type}", "b: {neon_type}", "c: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - FnCall: [simd_sub, [a, {FnCall: [simd_mul, [b, c]]}]] - - - name: "vcge{neon_type.no}" - doc: "Compare unsigned greater than or equal" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhs]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vcge{neon_type[0].no}" - doc: "Floating-point compare greater than or equal" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmge]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: [simd_ge, [a, b]] - - - name: "vclt{neon_type.no}" - doc: "Compare unsigned less than" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.{neon_type}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhi]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vtst{neon_type[0].no}" - doc: "Unsigned compare bitwise Test bits nonzero" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vtst]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmtst]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, u8x8, 'u8x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint8x16_t, u8x16, 'u8x16::new(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint16x4_t, u16x4, 'u16x4::new(0, 0, 0, 0)'] - - [uint16x8_t, u16x8, 'u16x8::new(0, 0, 0, 0, 0, 0, 0, 0)'] - - [uint32x2_t, u32x2, 'u32x2::new(0, 0)'] - - [uint32x4_t, u32x4, 'u32x4::new(0, 0, 0, 0)'] - compose: - - Let: [c, "{neon_type[0]}", {FnCall: [simd_and, [a, b]]}] - - Let: [d, "{type[1]}", "{type[2]}"] - - FnCall: [simd_ne, [c, {FnCall: [transmute, [d]]}]] - - - name: "vshl{neon_type[0].N}" - doc: "Shift left" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vshl, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [shl, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, '3'] - - [int8x16_t, '3'] - - [int16x4_t, '4'] - - [int16x8_t, '4'] - - [int32x2_t, '5'] - - [int32x4_t, '5'] - - [uint8x8_t, '3'] - - [uint8x16_t, '3'] - - [uint16x4_t, '4'] - - [uint16x8_t, '4'] - - [uint32x2_t, '5'] - - [uint32x4_t, '5'] - - [int64x1_t, '6'] - - [int64x2_t, '6'] - - [uint64x1_t, '6'] - - [uint64x2_t, '6'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[1]}"]] - - FnCall: - - simd_shl - - - a - - FnCall: ["vdup{neon_type[0].N}", ['N as _']] - - - name: "vsra{neon_type[0].N}" - doc: "Unsigned shift right and accumulate" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vsra, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [usra, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, '8'] - - [uint8x16_t, '8'] - - [uint16x4_t, '16'] - - [uint16x8_t, '16'] - - [uint32x2_t, '32'] - - [uint32x4_t, '32'] - - [uint64x1_t, '64'] - - [uint64x2_t, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[1]}']] - - FnCall: - - simd_add - - - a - - FnCall: ["vshr{neon_type[0].N}::", [b]] - - - name: "vrsra{neon_type[0].N}" - doc: "Unsigned rounding shift right and accumulate" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsra, 'N = 2']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ursra, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint8x8_t, '8'] - - [uint8x16_t, '8'] - - [uint16x4_t, '16'] - - [uint16x8_t, '16'] - - [uint32x2_t, '32'] - - [uint32x4_t, '32'] - - [uint64x1_t, '64'] - - [uint64x2_t, '64'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[1]}']] - - FnCall: - - simd_add - - - a - - FnCall: ["vrshr{neon_type[0].N}::", [b]] - - - name: "vqrshrn_n_{neon_type[0]}" - doc: "Unsigned signed saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, '8', 'const { uint16x8_t([-N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16, -N as u16]) }'] - - [uint32x4_t, uint16x4_t, '16', 'const { uint32x4_t([-N as u32, -N as u32, -N as u32, -N as u32]) }'] - - [uint64x2_t, uint32x2_t, '32', 'const { uint64x2_t([-N as u64, -N as u64]) }'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[2]}']] - - LLVMLink: - name: "vqrshrn{neon_type[0].N}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqrshiftnu.{neon_type[1]}" - arch: arm - - FnCall: ["_vqrshrn{neon_type[0].N}", ["a.as_signed()", "{type[3]}"]] - - - name: "vqrshrn_n_{neon_type[0]}" - doc: "Unsigned signed saturating rounded shift right narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-not-arm - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [uqrshrn, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t, '8'] - - [uint32x4_t, uint16x4_t, '16'] - - [uint64x2_t, uint32x2_t, '32'] - compose: - - FnCall: [static_assert!, ['N >= 1 && N <= {type[2]}']] - - LLVMLink: - name: "vqrshrn_n_{neon_type[0]}" - arguments: - - "a: {neon_type[0]}" - - "n: i32" - links: - - link: "llvm.aarch64.neon.uqrshrn.{neon_type[1]}" - arch: aarch64,arm64ec - - FnCall: ["_vqrshrn_n_{neon_type[0]}", ["a.as_signed()", N]] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to unsigned fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcvtzu]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - LLVMLink: - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" - arch: arm - - link: "llvm.fptoui.sat.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vmla{neon_type[0].N}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i16"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", int16x4_t] - - [int16x8_t, "i16", int16x8_t] - - [uint16x4_t, "u16", uint16x4_t] - - [uint16x8_t, "u16", uint16x8_t] - compose: - - FnCall: - - "vmla{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [c]] - - - name: "vmla{neon_type[0].N}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, "i32", int32x2_t] - - [int32x4_t, "i32", int32x4_t] - - [uint32x2_t, "u32", uint32x2_t] - - [uint32x4_t, "u32", uint32x4_t] - compose: - - FnCall: - - "vmla{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [c]] - - - name: "vmla{neon_type[0].N}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, "f32", float32x2_t] - - [float32x4_t, "f32", float32x4_t] - compose: - - FnCall: ["vmla{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [c]]}]] - - - name: "vmla{type[0]}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i16"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_u16, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_u16, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmla{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmla{type[0]}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.i32"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mla, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_u32, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_u32, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmla{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmla{type[0]}" - doc: "Vector multiply accumulate with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmla.f32"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_f32, float32x2_t, float32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_f32, float32x2_t, float32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_f32, float32x4_t, float32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmla{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmls{neon_type[0].N}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i16"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16", int16x4_t] - - [int16x8_t, "i16", int16x8_t] - - [uint16x4_t, "u16", uint16x4_t] - - [uint16x8_t, "u16", uint16x8_t] - compose: - - FnCall: - - "vmls{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [c]] - - - name: "vmls{neon_type[0].N}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int32x2_t, "i32", int32x2_t] - - [int32x4_t, "i32", int32x4_t] - - [uint32x2_t, "u32", uint32x2_t] - - [uint32x4_t, "u32", uint32x4_t] - compose: - - FnCall: - - "vmls{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}", [c]] - - - name: "vmls{neon_type[0].N}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, "f32", float32x2_t] - - [float32x4_t, "f32", float32x4_t] - compose: - - FnCall: ["vmls{neon_type[0].no}", [a, b, {FnCall: ["vdup{neon_type[0].N}", [c]]}]] - - - name: "vmls{type[0]}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i16"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_u16, uint16x4_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_u16, uint16x4_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_u16, uint16x8_t, uint16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_u16, uint16x8_t, uint16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmls{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmls{type[0]}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.i32"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mls, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_u32, uint32x2_t, uint32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_u32, uint32x2_t, uint32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_u32, uint32x4_t, uint32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_u32, uint32x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmls{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmls{type[0]}" - doc: "Vector multiply subtract with scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[1]}", "c: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vmls.f32"', 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['3']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_f32, float32x2_t, float32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_f32, float32x2_t, float32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_f32, float32x4_t, float32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_f32, float32x4_t, float32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - "vmls{neon_type[1].no}" - - - a - - b - - FnCall: [simd_shuffle!, [c, c, "{type[4]}"]] - - - name: "vmul{neon_type[0].N}" - doc: "Vector multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [mul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16"] - - [int16x8_t, "i16"] - - [int32x2_t, "i32"] - - [int32x4_t, "i32"] - - [uint16x4_t, "u16"] - - [uint16x8_t, "u16"] - - [uint32x2_t, "u32"] - - [uint32x4_t, "u32"] - compose: - - FnCall: - - simd_mul - - - a - - FnCall: ["vdup{neon_type[0].N}", [b]] - - - name: "vmul{neon_type[0].N}" - doc: "Vector multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, "f32"] - - [float32x4_t, "f32"] - compose: - - FnCall: - - simd_mul - - - a - - FnCall: ["vdup{neon_type[0].N}", [b]] - - - name: "vmul{type[2]}" - doc: "Floating-point multiply" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmul, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmul, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [float32x2_t, float32x2_t, '_lane_f32', '1', '[LANE as u32, LANE as u32]'] - - [float32x2_t, float32x4_t, '_laneq_f32', '2', '[LANE as u32, LANE as u32]'] - - [float32x4_t, float32x2_t, 'q_lane_f32', '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [float32x4_t, float32x4_t, 'q_laneq_f32', '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, '{type[3]}']] - - FnCall: - - simd_mul - - - a - - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] - - - name: "vqrdmulh{type[0]}" - doc: "Vector rounding saturating doubling multiply high by scalar" - arguments: ["a: {neon_type[1]}", "b: {neon_type[2]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh, 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [_lane_s16, int16x4_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_laneq_s16, int16x4_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_lane_s16, int16x8_t, int16x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s16, int16x8_t, int16x8_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [_lane_s32, int32x2_t, int32x2_t, '1', '[LANE as u32, LANE as u32]'] - - [_laneq_s32, int32x2_t, int32x4_t, '2', '[LANE as u32, LANE as u32]'] - - [q_lane_s32, int32x4_t, int32x2_t, '1', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [q_laneq_s32, int32x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - Let: [b, "{neon_type[1]}", {FnCall: [simd_shuffle!, [b, b, '{type[4]}']]}] - - FnCall: ["vqrdmulh{neon_type[1].no}", [a, b]] - - - name: "vqrdmulh{neon_type[0].N}" - doc: "Vector saturating rounding doubling multiply high with scalar" - arguments: ["a: {neon_type[0]}", "b: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqrdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqrdmulh]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [int16x4_t, "i16"] - - [int16x8_t, "i16"] - - [int32x2_t, "i32"] - - [int32x4_t, "i32"] - compose: - - FnCall: - - "vqrdmulh{neon_type[0].no}" - - - a - - FnCall: ["vdup{neon_type[0].N}", [b]] - - - name: "vclt{neon_type[0].no}" - doc: "Floating-point compare less than" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcgt.f32"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcmgt]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, uint32x2_t] - - [float32x4_t, uint32x4_t] - compose: - - FnCall: [simd_lt, [a, b]] - - - name: "vabdl_{neon_type[0]}" - doc: "Unsigned Absolute difference Long" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vabdl.{neon_type[0]}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uabdl]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint8x8_t, uint16x8_t] - - [uint16x4_t, uint32x4_t] - - [uint32x2_t, uint64x2_t] - compose: - - FnCall: [simd_cast, [{FnCall: ["vabd_{neon_type[0]}", [a, b]]}]] - - - name: "vmull_lane{neon_type[1].no}" - doc: "Vector long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmull, 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [smull, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int16x4_t, int16x4_t, int32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int16x4_t, int16x8_t, int32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [int32x2_t, int32x2_t, int64x2_t, '1', '[LANE as u32, LANE as u32]'] - - [int32x2_t, int32x4_t, int64x2_t, '2', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmull_{neon_type[0]}" - - - a - - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] - - - name: "vmull_lane{neon_type[1].no}" - doc: "Vector long multiply by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[2]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vmull, 'LANE = 1']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [umull, 'LANE = 1']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [uint16x4_t, uint16x4_t, uint32x4_t, '2', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint16x4_t, uint16x8_t, uint32x4_t, '3', '[LANE as u32, LANE as u32, LANE as u32, LANE as u32]'] - - [uint32x2_t, uint32x2_t, uint64x2_t, '1', '[LANE as u32, LANE as u32]'] - - [uint32x2_t, uint32x4_t, uint64x2_t, '2', '[LANE as u32, LANE as u32]'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[3]}"]] - - FnCall: - - "vmull_{neon_type[0]}" - - - a - - FnCall: [simd_shuffle!, [b, b, "{type[4]}"]] - - - name: "vfms{neon_type[0].N}" - doc: "Floating-point fused Multiply-subtract to accumulator(vector)" - arguments: ["a: {neon_type[0]}", "b: {neon_type[0]}", "c: {type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "vfp4"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vfms]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fmls]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, "f32"] - - [float32x4_t, "f32"] - compose: - - FnCall: - - "vfms{neon_type[0].no}" - - - a - - b - - FnCall: ["vdup{neon_type[0].N}_vfp4", [c]] - - - name: "vqdmulh{neon_type[0].laneq_nox}" - doc: "Vector saturating doubling multiply high by scalar" - arguments: ["a: {neon_type[0]}", "b: {neon_type[1]}"] - return_type: "{neon_type[0]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqdmulh, 'LANE = 0']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [sqdmulh, 'LANE = 0']]}]] - - FnCall: [rustc_legacy_const_generics, ['2']] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - static_defs: ['const LANE: i32'] - safety: - unsafe: [neon] - types: - - [int16x8_t, int16x8_t, '3'] - - [int16x4_t, int16x8_t, '3'] - - [int32x4_t, int32x4_t, '2'] - - [int32x2_t, int32x4_t, '2'] - compose: - - FnCall: [static_assert_uimm_bits!, [LANE, "{type[2]}"]] - - FnCall: - - "vqdmulh{neon_type[0].no}" - - - a - - FnCall: - - "vdup{neon_type[0].N}" - - - FnCall: [simd_extract!, [b, 'LANE as u32']] - - - name: "vrecpe{neon_type.no}" - doc: "Unsigned reciprocal estimate" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrecpe]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [urecpe]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "vrecpe{neon_type.no}" - links: - - link: "llvm.arm.neon.vrecpe.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.urecpe.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrsqrte{neon_type.no}" - doc: "Unsigned reciprocal square root estimate" - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrte]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ursqrte]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint32x2_t - - uint32x4_t - compose: - - LLVMLink: - name: "vrsqrte{neon_type.no}" - links: - - link: "llvm.arm.neon.vrsqrte.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.ursqrte.{neon_type}" - arch: aarch64,arm64ec - - - name: "vrsqrte{neon_type.no}" - doc: "Reciprocal square-root estimate." - arguments: ["a: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vrsqrte]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [frsqrte]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - float32x2_t - - float32x4_t - compose: - - LLVMLink: - name: "vrsqrte{neon_type.no}" - links: - - link: "llvm.arm.neon.vrsqrte.{neon_type}" - arch: arm - - link: "llvm.aarch64.neon.frsqrte.{neon_type}" - arch: aarch64,arm64ec - - - name: "vqshlu{neon_type[0].N}" - doc: "Signed saturating shift left unsigned" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - *target-is-arm - - *enable-v7 - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vqshlu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, '3', 'const { int8x8_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] - - [int16x4_t, uint16x4_t, '4', 'const { int16x4_t([N as i16, N as i16, N as i16, N as i16]) }'] - - [int32x2_t, uint32x2_t, '5', 'const { int32x2_t([N as i32, N as i32]) }'] - - [int64x1_t, uint64x1_t, '6', 'const { int64x1_t([N as i64]) }'] - - [int8x16_t, uint8x16_t, '3', 'const { int8x16_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] - - [int16x8_t, uint16x8_t, '4', 'const { int16x8_t([N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16]) }'] - - [int32x4_t, uint32x4_t, '5', 'const { int32x4_t([N as i32, N as i32, N as i32, N as i32]) }'] - - [int64x2_t, uint64x2_t, '6', 'const { int64x2_t([N as i64, N as i64]) }'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] - - LLVMLink: - name: "vqshlu{neon_type[0].N}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.arm.neon.vqshiftsu.{neon_type[0]}" - arch: arm - - FnCall: ["_vqshlu{neon_type[0].N}", [a, "{type[3]}"]] - - - name: "vqshlu{neon_type[0].N}" - doc: "Signed saturating shift left unsigned" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [sqshlu, 'N = 2']]}]] - - FnCall: [rustc_legacy_const_generics, ['1']] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - static_defs: ['const N: i32'] - safety: - unsafe: [neon] - types: - - [int8x8_t, uint8x8_t, '3', 'const { int8x8_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] - - [int16x4_t, uint16x4_t, '4', 'const { int16x4_t([N as i16, N as i16, N as i16, N as i16]) }'] - - [int32x2_t, uint32x2_t, '5', 'const { int32x2_t([N as i32, N as i32]) }'] - - [int64x1_t, uint64x1_t, '6', 'const { int64x1_t([N as i64]) }'] - - [int8x16_t, uint8x16_t, '3', 'const { int8x16_t([N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8, N as i8]) }'] - - [int16x8_t, uint16x8_t, '4', 'const { int16x8_t([N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16, N as i16]) }'] - - [int32x4_t, uint32x4_t, '5', 'const { int32x4_t([N as i32, N as i32, N as i32, N as i32]) }'] - - [int64x2_t, uint64x2_t, '6', 'const { int64x2_t([N as i64, N as i64]) }'] - compose: - - FnCall: [static_assert_uimm_bits!, [N, "{type[2]}"]] - - LLVMLink: - name: "vqshlu{neon_type[0].N}" - arguments: - - "a: {neon_type[0]}" - - "n: {neon_type[0]}" - links: - - link: "llvm.aarch64.neon.sqshlu.{neon_type[0]}" - arch: aarch64,arm64ec - - FnCall: ["_vqshlu{neon_type[0].N}", [a, "{type[3]}"]] - - - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - doc: "Floating-point convert to signed fixed-point, rounding toward zero" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vcvt]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [fcvtzs]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [float32x2_t, int32x2_t] - - [float32x4_t, int32x4_t] - compose: - - LLVMLink: - name: "vcvt{neon_type[1].no}_{neon_type[0]}" - links: - - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" - arch: arm - - link: "llvm.fptosi.sat.{neon_type[1]}.{neon_type[0]}" - arch: aarch64,arm64ec - - - name: "vqmovn_{neon_type[0]}" - doc: "Unsigned saturating extract narrow" - arguments: ["a: {neon_type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vqmovn]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [uqxtn]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - [uint16x8_t, uint8x8_t] - - [uint32x4_t, uint16x4_t] - - [uint64x2_t, uint32x2_t] - compose: - - LLVMLink: - name: "vqmovn_{neon_type[1]}" - links: - - link: "llvm.arm.neon.vqmovnu.{neon_type[1]}" - arch: arm - - link: "llvm.aarch64.neon.uqxtn.{neon_type[1]}" - arch: aarch64,arm64ec - - - name: "vcle{neon_type.no}" - doc: "Compare unsigned less than or equal" - arguments: ["a: {neon_type}", "b: {neon_type}"] - return_type: "{neon_type}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, ['"vcge.{neon_type}"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [cmhs]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - uint8x8_t - - uint8x16_t - - uint16x4_t - - uint16x8_t - - uint32x2_t - - uint32x4_t - compose: - - FnCall: [simd_le, [a, b]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vld4]]}]] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x4_t, int8x8_t, '1'] - - ["*const i16", int16x4x4_t, int16x4_t, '2'] - - ["*const i32", int32x2x4_t, int32x2_t, '4'] - - ["*const i8", int8x16x4_t, int8x16_t, '1'] - - ["*const i16", int16x8x4_t, int16x8_t, '2'] - - ["*const i32", int32x4x4_t, int32x4_t, '4'] - - ["*const f32", float32x2x4_t, float32x2_t, '4'] - - ["*const f32", float32x4x4_t, float32x4_t, '4'] - compose: - - LLVMLink: - name: "vld4{neon_type[1].dup_nox}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld4dup.{neon_type[2]}.p0i8" - arch: arm - - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as *const i8', "{type[3]}"]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, [{FnCall: [not, ['target_arch = "arm"']]}]] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']] - safety: - unsafe: [neon] - types: - - ["*const i8", int8x8x4_t, int8x8_t, 'p0i8'] - - ["*const i16", int16x4x4_t, int16x4_t, 'p0i16'] - - ["*const i32", int32x2x4_t, int32x2_t, 'p0i32'] - - ["*const i8", int8x16x4_t, int8x16_t, 'p0i8'] - - ["*const i16", int16x8x4_t, int16x8_t, 'p0i16'] - - ["*const i32", int32x4x4_t, int32x4_t, 'p0i32'] - - ["*const i64", int64x1x4_t, int64x1_t, 'p0i64'] - - ["*const f32", float32x2x4_t, float32x2_t, 'p0f32'] - - ["*const f32", float32x4x4_t, float32x4_t, 'p0f32'] - compose: - - LLVMLink: - name: "vld4{neon_type[1].dup_nox}" - arguments: - - "ptr: {type[0]}" - links: - - link: "llvm.aarch64.neon.ld4r.{neon_type[2]}.{type[3]}" - arch: aarch64,arm64ec - - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as _']] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg, ['target_arch = "arm"']] - - FnCall: [target_feature, ['enable = "neon,v7"']] - - FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']] - safety: - unsafe: [neon] - types: - - ["*const i64", int64x1x4_t] - compose: - - LLVMLink: - name: "vld4{neon_type[1].dup_nox}" - arguments: - - "ptr: *const i8" - - "size: i32" - links: - - link: "llvm.arm.neon.vld4dup.v1i64.p0i8" - arch: arm - - FnCall: ["_vld4{neon_type[1].dup_nox}", ['a as *const i8', '8']] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [vld4]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*const u8", uint8x8x4_t, int8x8x4_t] - - ["*const u16", uint16x4x4_t, int16x4x4_t] - - ["*const u32", uint32x2x4_t, int32x2x4_t] - - ["*const u8", uint8x16x4_t, int8x16x4_t] - - ["*const u16", uint16x8x4_t, int16x8x4_t] - - ["*const u32", uint32x4x4_t, int32x4x4_t] - - ["*const p8", poly8x8x4_t, int8x8x4_t] - - ["*const p16", poly16x4x4_t, int16x4x4_t] - - ["*const p8", poly8x16x4_t, int8x16x4_t] - - ["*const p16", poly16x8x4_t, int16x8x4_t] - compose: - - FnCall: - - "transmute" - - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v7"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*const u64", uint64x1x4_t, int64x1x4_t] - compose: - - FnCall: - - "transmute" - - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] - - - name: "vld4{neon_type[1].dup_nox}" - doc: "Load single 4-element structure and replicate to all lanes of four registers" - arguments: ["a: {type[0]}"] - return_type: "{neon_type[1]}" - attr: - - FnCall: [target_feature, ['enable = "neon,aes"']] - - FnCall: [cfg_attr, [target_arch = "arm", {FnCall: [target_feature, ['enable = "v8"']]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, 'target_arch = "arm"']]}, {FnCall: [assert_instr, [nop]]}]] - - FnCall: [cfg_attr, [{FnCall: [all, [test, {FnCall: [any, ['target_arch = "aarch64"', 'target_arch = "arm64ec"']]}]]}, {FnCall: [assert_instr, [ld4r]]}]] - - FnCall: [cfg_attr, [{FnCall: [not, ['target_arch = "arm"']]}, {FnCall: [stable, ['feature = "neon_intrinsics"', 'since = "1.59.0"']]}]] - - FnCall: [cfg_attr, ['target_arch = "arm"', {FnCall: [unstable, ['feature = "stdarch_arm_neon_intrinsics"', 'issue = "111800"']]}]] - safety: - unsafe: [neon] - types: - - ["*const p64", poly64x1x4_t, int64x1x4_t] - compose: - - FnCall: - - "transmute" - - - FnCall: ["vld4{neon_type[2].dup_nox}", [{FnCall: [transmute, [a]]}]] - diff --git a/library/stdarch/crates/stdarch-gen2/src/assert_instr.rs b/library/stdarch/crates/stdarch-gen2/src/assert_instr.rs deleted file mode 100644 index ce1bbe8b55f..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/assert_instr.rs +++ /dev/null @@ -1,372 +0,0 @@ -use proc_macro2::TokenStream; -use quote::{format_ident, quote, ToTokens, TokenStreamExt}; -use serde::de::{self, MapAccess, Visitor}; -use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; -use std::fmt; - -use crate::{ - context::{self, Context}, - typekinds::{BaseType, BaseTypeKind}, - wildstring::WildString, -}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum InstructionAssertion { - Basic(WildString), - WithArgs(WildString, WildString), -} - -impl InstructionAssertion { - fn build(&mut self, ctx: &Context) -> context::Result { - match self { - InstructionAssertion::Basic(ws) => ws.build_acle(ctx.local), - InstructionAssertion::WithArgs(ws, args_ws) => [ws, args_ws] - .into_iter() - .try_for_each(|ws| ws.build_acle(ctx.local)), - } - } -} - -impl ToTokens for InstructionAssertion { - fn to_tokens(&self, tokens: &mut TokenStream) { - let instr = format_ident!( - "{}", - match self { - Self::Basic(instr) => instr, - Self::WithArgs(instr, _) => instr, - } - .to_string() - ); - tokens.append_all(quote! { #instr }); - - if let Self::WithArgs(_, args) = self { - let ex: TokenStream = args - .to_string() - .parse() - .expect("invalid instruction assertion arguments expression given"); - tokens.append_all(quote! {, #ex}) - } - } -} - -// Asserts that the given instruction is present for the intrinsic of the associated type bitsize. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(remote = "Self")] -pub struct InstructionAssertionMethodForBitsize { - pub default: InstructionAssertion, - pub byte: Option, - pub halfword: Option, - pub word: Option, - pub doubleword: Option, -} - -impl InstructionAssertionMethodForBitsize { - fn build(&mut self, ctx: &Context) -> context::Result { - if let Some(ref mut byte) = self.byte { - byte.build(ctx)? - } - if let Some(ref mut halfword) = self.halfword { - halfword.build(ctx)? - } - if let Some(ref mut word) = self.word { - word.build(ctx)? - } - if let Some(ref mut doubleword) = self.doubleword { - doubleword.build(ctx)? - } - self.default.build(ctx) - } -} - -impl Serialize for InstructionAssertionMethodForBitsize { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - InstructionAssertionMethodForBitsize { - default: InstructionAssertion::Basic(instr), - byte: None, - halfword: None, - word: None, - doubleword: None, - } => serializer.serialize_str(&instr.to_string()), - InstructionAssertionMethodForBitsize { - default: InstructionAssertion::WithArgs(instr, args), - byte: None, - halfword: None, - word: None, - doubleword: None, - } => { - let mut seq = serializer.serialize_seq(Some(2))?; - seq.serialize_element(&instr.to_string())?; - seq.serialize_element(&args.to_string())?; - seq.end() - } - _ => InstructionAssertionMethodForBitsize::serialize(self, serializer), - } - } -} - -impl<'de> Deserialize<'de> for InstructionAssertionMethodForBitsize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct IAMVisitor; - - impl<'de> Visitor<'de> for IAMVisitor { - type Value = InstructionAssertionMethodForBitsize; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("array, string or map") - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - Ok(InstructionAssertionMethodForBitsize { - default: InstructionAssertion::Basic(value.parse().map_err(E::custom)?), - byte: None, - halfword: None, - word: None, - doubleword: None, - }) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: de::SeqAccess<'de>, - { - use serde::de::Error; - let make_err = - || Error::custom("invalid number of arguments passed to assert_instruction"); - let instruction = seq.next_element()?.ok_or_else(make_err)?; - let args = seq.next_element()?.ok_or_else(make_err)?; - - if let Some(true) = seq.size_hint().map(|len| len > 0) { - Err(make_err()) - } else { - Ok(InstructionAssertionMethodForBitsize { - default: InstructionAssertion::WithArgs(instruction, args), - byte: None, - halfword: None, - word: None, - doubleword: None, - }) - } - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - InstructionAssertionMethodForBitsize::deserialize( - de::value::MapAccessDeserializer::new(map), - ) - } - } - - deserializer.deserialize_any(IAMVisitor) - } -} - -/// Asserts that the given instruction is present for the intrinsic of the associated type. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(remote = "Self")] -pub struct InstructionAssertionMethod { - /// Instruction for integer intrinsics - pub default: InstructionAssertionMethodForBitsize, - /// Instruction for floating-point intrinsics (optional) - #[serde(default)] - pub float: Option, - /// Instruction for unsigned integer intrinsics (optional) - #[serde(default)] - pub unsigned: Option, -} - -impl InstructionAssertionMethod { - pub(crate) fn build(&mut self, ctx: &Context) -> context::Result { - if let Some(ref mut float) = self.float { - float.build(ctx)? - } - if let Some(ref mut unsigned) = self.unsigned { - unsigned.build(ctx)? - } - self.default.build(ctx) - } -} - -impl Serialize for InstructionAssertionMethod { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - InstructionAssertionMethod { - default: - InstructionAssertionMethodForBitsize { - default: InstructionAssertion::Basic(instr), - byte: None, - halfword: None, - word: None, - doubleword: None, - }, - float: None, - unsigned: None, - } => serializer.serialize_str(&instr.to_string()), - InstructionAssertionMethod { - default: - InstructionAssertionMethodForBitsize { - default: InstructionAssertion::WithArgs(instr, args), - byte: None, - halfword: None, - word: None, - doubleword: None, - }, - float: None, - unsigned: None, - } => { - let mut seq = serializer.serialize_seq(Some(2))?; - seq.serialize_element(&instr.to_string())?; - seq.serialize_element(&args.to_string())?; - seq.end() - } - _ => InstructionAssertionMethod::serialize(self, serializer), - } - } -} - -impl<'de> Deserialize<'de> for InstructionAssertionMethod { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct IAMVisitor; - - impl<'de> Visitor<'de> for IAMVisitor { - type Value = InstructionAssertionMethod; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("array, string or map") - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - Ok(InstructionAssertionMethod { - default: InstructionAssertionMethodForBitsize { - default: InstructionAssertion::Basic(value.parse().map_err(E::custom)?), - byte: None, - halfword: None, - word: None, - doubleword: None, - }, - float: None, - unsigned: None, - }) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: de::SeqAccess<'de>, - { - use serde::de::Error; - let make_err = - || Error::custom("invalid number of arguments passed to assert_instruction"); - let instruction = seq.next_element()?.ok_or_else(make_err)?; - let args = seq.next_element()?.ok_or_else(make_err)?; - - if let Some(true) = seq.size_hint().map(|len| len > 0) { - Err(make_err()) - } else { - Ok(InstructionAssertionMethod { - default: InstructionAssertionMethodForBitsize { - default: InstructionAssertion::WithArgs(instruction, args), - byte: None, - halfword: None, - word: None, - doubleword: None, - }, - float: None, - unsigned: None, - }) - } - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - InstructionAssertionMethod::deserialize(de::value::MapAccessDeserializer::new(map)) - } - } - - deserializer.deserialize_any(IAMVisitor) - } -} - -#[derive(Debug)] -pub struct InstructionAssertionsForBaseType<'a>( - pub &'a Vec, - pub &'a Option<&'a BaseType>, -); - -impl<'a> ToTokens for InstructionAssertionsForBaseType<'a> { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.0.iter().for_each( - |InstructionAssertionMethod { - default, - float, - unsigned, - }| { - let kind = self.1.map(|ty| ty.kind()); - let instruction = match (kind, float, unsigned) { - (None, float, unsigned) if float.is_some() || unsigned.is_some() => { - unreachable!( - "cannot determine the base type kind for instruction assertion: {self:#?}") - } - (Some(BaseTypeKind::Float), Some(float), _) => float, - (Some(BaseTypeKind::UInt), _, Some(unsigned)) => unsigned, - _ => default, - }; - - let bitsize = self.1.and_then(|ty| ty.get_size().ok()); - let instruction = match (bitsize, instruction) { - ( - Some(8), - InstructionAssertionMethodForBitsize { - byte: Some(byte), .. - }, - ) => byte, - ( - Some(16), - InstructionAssertionMethodForBitsize { - halfword: Some(halfword), - .. - }, - ) => halfword, - ( - Some(32), - InstructionAssertionMethodForBitsize { - word: Some(word), .. - }, - ) => word, - ( - Some(64), - InstructionAssertionMethodForBitsize { - doubleword: Some(doubleword), - .. - }, - ) => doubleword, - (_, InstructionAssertionMethodForBitsize { default, .. }) => default, - }; - - tokens.append_all(quote! { #[cfg_attr(test, assert_instr(#instruction))]}) - }, - ); - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/context.rs b/library/stdarch/crates/stdarch-gen2/src/context.rs deleted file mode 100644 index 8405428b7ad..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/context.rs +++ /dev/null @@ -1,263 +0,0 @@ -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::{collections::HashMap, usize}; - -use crate::{ - expression::Expression, - input::{InputSet, InputType}, - intrinsic::{Constraint, Intrinsic, Signature}, - matching::SizeMatchable, - predicate_forms::PredicateForm, - typekinds::{ToRepr, TypeKind}, - wildcards::Wildcard, - wildstring::WildString, -}; - -/// Maximum SVE vector size -const SVE_VECTOR_MAX_SIZE: u32 = 2048; -/// Vector register size -const VECTOR_REG_SIZE: u32 = 128; - -/// Generator result -pub type Result = std::result::Result; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ArchitectureSettings { - #[serde(alias = "arch")] - pub arch_name: String, - pub target_feature: Vec, - #[serde(alias = "llvm_prefix")] - pub llvm_link_prefix: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GlobalContext { - pub arch_cfgs: Vec, - #[serde(default)] - pub uses_neon_types: bool, -} - -/// Context of an intrinsic group -#[derive(Debug, Clone, Default)] -pub struct GroupContext { - /// LLVM links to target input sets - pub links: HashMap, -} - -#[derive(Debug, Clone, Copy)] -pub enum VariableType { - Argument, - Internal, -} - -#[derive(Debug, Clone)] -pub struct LocalContext { - pub signature: Signature, - - pub input: InputSet, - - pub substitutions: HashMap, - pub variables: HashMap, -} - -impl LocalContext { - pub fn new(input: InputSet, original: &Intrinsic) -> LocalContext { - LocalContext { - signature: original.signature.clone(), - input, - substitutions: HashMap::new(), - variables: HashMap::new(), - } - } - - pub fn provide_type_wildcard(&self, wildcard: &Wildcard) -> Result { - let err = || { - format!( - "provide_type_wildcard() wildcard {{{wildcard}}} not found for {}", - &self.signature.name.to_string() - ) - }; - - /* If the type is already a vector then we can just return the vector */ - let make_neon = |tuple_size| { - move |ty| match ty { - TypeKind::Vector(_) => Ok(ty), - _ => TypeKind::make_vector(ty, false, tuple_size), - } - }; - let make_sve = |tuple_size| move |ty| TypeKind::make_vector(ty, true, tuple_size); - - match wildcard { - Wildcard::Type(idx) => self.input.typekind(*idx).ok_or_else(err), - Wildcard::NEONType(idx, tuple_size, _) => self - .input - .typekind(*idx) - .ok_or_else(|| { - dbg!("{:?}", &self); - err() - }) - .and_then(make_neon(*tuple_size)), - Wildcard::SVEType(idx, tuple_size) => self - .input - .typekind(*idx) - .ok_or_else(err) - .and_then(make_sve(*tuple_size)), - Wildcard::Predicate(idx) => self.input.typekind(*idx).map_or_else( - || { - if idx.is_none() && self.input.types_len() == 1 { - Err(err()) - } else { - Err(format!( - "there is no type at index {} to infer the predicate from", - idx.unwrap_or(0) - )) - } - }, - |ref ty| TypeKind::make_predicate_from(ty), - ), - Wildcard::MaxPredicate => self - .input - .iter() - .filter_map(|arg| arg.typekind()) - .max_by(|x, y| { - x.base_type() - .and_then(|bt| bt.get_size().ok()) - .unwrap_or(0) - .cmp(&y.base_type().and_then(|bt| bt.get_size().ok()).unwrap_or(0)) - }) - .map_or_else( - || Err("there are no types available to infer the predicate from".to_string()), - TypeKind::make_predicate_from, - ), - Wildcard::Scale(w, as_ty) => { - let mut ty = self.provide_type_wildcard(w)?; - if let Some(vty) = ty.vector_mut() { - let base_ty = if let Some(w) = as_ty.wildcard() { - *self.provide_type_wildcard(w)?.base_type().unwrap() - } else { - *as_ty.base_type().unwrap() - }; - vty.cast_base_type_as(base_ty) - } - Ok(ty) - } - _ => Err(err()), - } - } - - pub fn provide_substitution_wildcard(&self, wildcard: &Wildcard) -> Result { - let err = || Err(format!("wildcard {{{wildcard}}} not found")); - - match wildcard { - Wildcard::SizeLiteral(idx) => self.input.typekind(*idx) - .map_or_else(err, |ty| Ok(ty.size_literal())), - Wildcard::Size(idx) => self.input.typekind(*idx) - .map_or_else(err, |ty| Ok(ty.size())), - Wildcard::SizeMinusOne(idx) => self.input.typekind(*idx) - .map_or_else(err, |ty| Ok((ty.size().parse::().unwrap()-1).to_string())), - Wildcard::SizeInBytesLog2(idx) => self.input.typekind(*idx) - .map_or_else(err, |ty| Ok(ty.size_in_bytes_log2())), - Wildcard::NVariant if self.substitutions.get(wildcard).is_none() => Ok(String::new()), - Wildcard::TypeKind(idx, opts) => { - self.input.typekind(*idx) - .map_or_else(err, |ty| { - let literal = if let Some(opts) = opts { - opts.contains(ty.base_type().map(|bt| *bt.kind()).ok_or_else(|| { - format!("cannot retrieve a type literal out of {ty}") - })?) - .then(|| ty.type_kind()) - .unwrap_or_default() - } else { - ty.type_kind() - }; - Ok(literal) - }) - } - Wildcard::PredicateForms(_) => self - .input - .iter() - .find_map(|arg| { - if let InputType::PredicateForm(pf) = arg { - Some(pf.get_suffix().to_string()) - } else { - None - } - }) - .ok_or_else(|| unreachable!("attempting to render a predicate form wildcard, but no predicate form was compiled for it")), - _ => self - .substitutions - .get(wildcard) - .map_or_else(err, |s| Ok(s.clone())), - } - } - - pub fn make_assertion_from_constraint(&self, constraint: &Constraint) -> Result { - match constraint { - Constraint::AnyI32 { - variable, - any_values, - } => { - let where_ex = any_values - .iter() - .map(|value| format!("{variable} == {value}")) - .join(" || "); - Ok(Expression::MacroCall("static_assert".to_string(), where_ex)) - } - Constraint::RangeI32 { - variable, - range: SizeMatchable::Matched(range), - } => Ok(Expression::MacroCall( - "static_assert_range".to_string(), - format!( - "{variable}, {min}, {max}", - min = range.start(), - max = range.end() - ), - )), - Constraint::SVEMaxElems { - variable, - sve_max_elems_type: ty, - } - | Constraint::VecMaxElems { - variable, - vec_max_elems_type: ty, - } => { - if !self.input.is_empty() { - let higher_limit = match constraint { - Constraint::SVEMaxElems { .. } => SVE_VECTOR_MAX_SIZE, - Constraint::VecMaxElems { .. } => VECTOR_REG_SIZE, - _ => unreachable!(), - }; - - let max = ty.base_type() - .map(|ty| ty.get_size()) - .transpose()? - .map_or_else( - || Err(format!("can't make an assertion out of constraint {self:?}: no valid type is present")), - |bitsize| Ok(higher_limit / bitsize - 1))?; - Ok(Expression::MacroCall( - "static_assert_range".to_string(), - format!("{variable}, 0, {max}"), - )) - } else { - Err(format!("can't make an assertion out of constraint {self:?}: no types are being used")) - } - } - _ => unreachable!("constraints were not built successfully!"), - } - } - - pub fn predicate_form(&self) -> Option<&PredicateForm> { - self.input.iter().find_map(|arg| arg.predicate_form()) - } - - pub fn n_variant_op(&self) -> Option<&WildString> { - self.input.iter().find_map(|arg| arg.n_variant_op()) - } -} - -pub struct Context<'ctx> { - pub local: &'ctx mut LocalContext, - pub group: &'ctx mut GroupContext, - pub global: &'ctx GlobalContext, -} diff --git a/library/stdarch/crates/stdarch-gen2/src/expression.rs b/library/stdarch/crates/stdarch-gen2/src/expression.rs deleted file mode 100644 index 83984679588..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/expression.rs +++ /dev/null @@ -1,576 +0,0 @@ -use itertools::Itertools; -use lazy_static::lazy_static; -use proc_macro2::{Literal, Punct, Spacing, TokenStream}; -use quote::{format_ident, quote, ToTokens, TokenStreamExt}; -use regex::Regex; -use serde::de::{self, MapAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize}; -use std::fmt; -use std::str::FromStr; - -use crate::intrinsic::Intrinsic; -use crate::{ - context::{self, Context, VariableType}, - intrinsic::{Argument, LLVMLink, StaticDefinition}, - matching::{MatchKindValues, MatchSizeValues}, - typekinds::{BaseType, BaseTypeKind, TypeKind}, - wildcards::Wildcard, - wildstring::WildString, -}; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub enum IdentifierType { - Variable, - Symbol, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum LetVariant { - Basic(WildString, Box), - WithType(WildString, TypeKind, Box), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FnCall( - /// Function pointer - pub Box, - /// Function arguments - pub Vec, - /// Function turbofish arguments - #[serde(default)] - pub Vec, -); - -impl FnCall { - pub fn new_expression(fn_ptr: Expression, arguments: Vec) -> Expression { - FnCall(Box::new(fn_ptr), arguments, Vec::new()).into() - } - - pub fn is_llvm_link_call(&self, llvm_link_name: &String) -> bool { - self.is_expected_call(llvm_link_name) - } - - pub fn is_target_feature_call(&self) -> bool { - self.is_expected_call("target_feature") - } - - pub fn is_expected_call(&self, fn_call_name: &str) -> bool { - if let Expression::Identifier(fn_name, IdentifierType::Symbol) = self.0.as_ref() { - &fn_name.to_string() == fn_call_name - } else { - false - } - } - - pub fn pre_build(&mut self, ctx: &mut Context) -> context::Result { - self.0.pre_build(ctx)?; - self.1 - .iter_mut() - .chain(self.2.iter_mut()) - .try_for_each(|ex| ex.pre_build(ctx)) - } - - pub fn build(&mut self, intrinsic: &Intrinsic, ctx: &mut Context) -> context::Result { - self.0.build(intrinsic, ctx)?; - self.1 - .iter_mut() - .chain(self.2.iter_mut()) - .try_for_each(|ex| ex.build(intrinsic, ctx)) - } -} - -impl ToTokens for FnCall { - fn to_tokens(&self, tokens: &mut TokenStream) { - let FnCall(fn_ptr, arguments, turbofish) = self; - - fn_ptr.to_tokens(tokens); - - if !turbofish.is_empty() { - tokens.append_all(quote! {::<#(#turbofish),*>}); - } - - tokens.append_all(quote! { (#(#arguments),*) }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(remote = "Self", deny_unknown_fields)] -pub enum Expression { - /// (Re)Defines a variable - Let(LetVariant), - /// Performs a variable assignment operation - Assign(String, Box), - /// Performs a macro call - MacroCall(String, String), - /// Performs a function call - FnCall(FnCall), - /// Performs a method call. The following: - /// `MethodCall: ["$object", "to_string", []]` - /// is tokenized as: - /// `object.to_string()`. - MethodCall(Box, String, Vec), - /// Symbol identifier name, prepend with a `$` to treat it as a scope variable - /// which engages variable tracking and enables inference. - /// E.g. `my_function_name` for a generic symbol or `$my_variable` for - /// a variable. - Identifier(WildString, IdentifierType), - /// Constant signed integer number expression - IntConstant(i32), - /// Constant floating point number expression - FloatConstant(f32), - /// Constant boolean expression, either `true` or `false` - BoolConstant(bool), - /// Array expression - Array(Vec), - - // complex expressions - /// Makes an LLVM link. - /// - /// It stores the link's function name in the wildcard `{llvm_link}`, for use in - /// subsequent expressions. - LLVMLink(LLVMLink), - /// Casts the given expression to the specified (unchecked) type - CastAs(Box, String), - /// Returns the LLVM `undef` symbol - SvUndef, - /// Multiplication - Multiply(Box, Box), - /// Xor - Xor(Box, Box), - /// Converts the specified constant to the specified type's kind - ConvertConst(TypeKind, i32), - /// Yields the given type in the Rust representation - Type(TypeKind), - - MatchSize(TypeKind, MatchSizeValues>), - MatchKind(TypeKind, MatchKindValues>), -} - -impl Expression { - pub fn pre_build(&mut self, ctx: &mut Context) -> context::Result { - match self { - Self::FnCall(fn_call) => fn_call.pre_build(ctx), - Self::MethodCall(cl_ptr_ex, _, arg_exs) => { - cl_ptr_ex.pre_build(ctx)?; - arg_exs.iter_mut().try_for_each(|ex| ex.pre_build(ctx)) - } - Self::Let(LetVariant::Basic(_, ex) | LetVariant::WithType(_, _, ex)) => { - ex.pre_build(ctx) - } - Self::CastAs(ex, _) => ex.pre_build(ctx), - Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { - lhs.pre_build(ctx)?; - rhs.pre_build(ctx) - } - Self::MatchSize(match_ty, values) => { - *self = *values.get(match_ty, ctx.local)?.to_owned(); - self.pre_build(ctx) - } - Self::MatchKind(match_ty, values) => { - *self = *values.get(match_ty, ctx.local)?.to_owned(); - self.pre_build(ctx) - } - _ => Ok(()), - } - } - - pub fn build(&mut self, intrinsic: &Intrinsic, ctx: &mut Context) -> context::Result { - match self { - Self::LLVMLink(link) => link.build_and_save(ctx), - Self::Identifier(identifier, id_type) => { - identifier.build_acle(ctx.local)?; - - if let IdentifierType::Variable = id_type { - ctx.local - .variables - .get(&identifier.to_string()) - .map(|_| ()) - .ok_or_else(|| format!("invalid variable {identifier} being referenced")) - } else { - Ok(()) - } - } - Self::FnCall(fn_call) => { - fn_call.build(intrinsic, ctx)?; - - if let Some(llvm_link_name) = ctx.local.substitutions.get(&Wildcard::LLVMLink) { - if fn_call.is_llvm_link_call(llvm_link_name) { - *self = intrinsic - .llvm_link() - .expect("got LLVMLink wildcard without a LLVM link in `compose`") - .apply_conversions_to_call(fn_call.clone(), ctx.local)? - } - } - - Ok(()) - } - Self::MethodCall(cl_ptr_ex, _, arg_exs) => { - cl_ptr_ex.build(intrinsic, ctx)?; - arg_exs - .iter_mut() - .try_for_each(|ex| ex.build(intrinsic, ctx)) - } - Self::Let(variant) => { - let (var_name, ex, ty) = match variant { - LetVariant::Basic(var_name, ex) => (var_name, ex, None), - LetVariant::WithType(var_name, ty, ex) => { - if let Some(w) = ty.wildcard() { - ty.populate_wildcard(ctx.local.provide_type_wildcard(w)?)?; - } - (var_name, ex, Some(ty.to_owned())) - } - }; - - var_name.build_acle(ctx.local)?; - ctx.local.variables.insert( - var_name.to_string(), - ( - ty.unwrap_or_else(|| TypeKind::Custom("unknown".to_string())), - VariableType::Internal, - ), - ); - ex.build(intrinsic, ctx) - } - Self::CastAs(ex, _) => ex.build(intrinsic, ctx), - Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { - lhs.build(intrinsic, ctx)?; - rhs.build(intrinsic, ctx) - } - Self::ConvertConst(ty, num) => { - if let Some(w) = ty.wildcard() { - *ty = ctx.local.provide_type_wildcard(w)? - } - - if let Some(BaseType::Sized(BaseTypeKind::Float, _)) = ty.base() { - *self = Expression::FloatConstant(*num as f32) - } else { - *self = Expression::IntConstant(*num) - } - Ok(()) - } - Self::Type(ty) => { - if let Some(w) = ty.wildcard() { - *ty = ctx.local.provide_type_wildcard(w)? - } - - Ok(()) - } - _ => Ok(()), - } - } - - /// True if the expression requires an `unsafe` context in a safe function. - /// - /// The classification is somewhat fuzzy, based on actual usage (e.g. empirical function names) - /// rather than a full parse. This is a reasonable approach because mistakes here will usually - /// be caught at build time: - /// - /// - Missing an `unsafe` is a build error. - /// - An unnecessary `unsafe` is a warning, made into an error by the CI's `-D warnings`. - /// - /// This **panics** if it encounters an expression that shouldn't appear in a safe function at - /// all (such as `SvUndef`). - pub fn requires_unsafe_wrapper(&self, ctx_fn: &str) -> bool { - match self { - // The call will need to be unsafe, but the declaration does not. - Self::LLVMLink(..) => false, - // Identifiers, literals and type names are never unsafe. - Self::Identifier(..) => false, - Self::IntConstant(..) => false, - Self::FloatConstant(..) => false, - Self::BoolConstant(..) => false, - Self::Type(..) => false, - Self::ConvertConst(..) => false, - // Nested structures that aren't inherently unsafe, but could contain other expressions - // that might be. - Self::Assign(_var, exp) => exp.requires_unsafe_wrapper(ctx_fn), - Self::Let(LetVariant::Basic(_, exp) | LetVariant::WithType(_, _, exp)) => { - exp.requires_unsafe_wrapper(ctx_fn) - } - Self::Array(exps) => exps.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)), - Self::Multiply(lhs, rhs) | Self::Xor(lhs, rhs) => { - lhs.requires_unsafe_wrapper(ctx_fn) || rhs.requires_unsafe_wrapper(ctx_fn) - } - Self::CastAs(exp, _ty) => exp.requires_unsafe_wrapper(ctx_fn), - // Functions and macros can be unsafe, but can also contain other expressions. - Self::FnCall(FnCall(fn_exp, args, turbo_args)) => { - let fn_name = fn_exp.to_string(); - fn_exp.requires_unsafe_wrapper(ctx_fn) - || fn_name.starts_with("_sv") - || fn_name.starts_with("simd_") - || fn_name.ends_with("transmute") - || args.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) - || turbo_args - .iter() - .any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) - } - Self::MethodCall(exp, fn_name, args) => match fn_name.as_str() { - // `as_signed` and `as_unsigned` are unsafe because they're trait methods with - // target features to allow use on feature-dependent types (such as SVE vectors). - // We can safely wrap them here. - "as_signed" => true, - "as_unsigned" => true, - _ => { - exp.requires_unsafe_wrapper(ctx_fn) - || args.iter().any(|exp| exp.requires_unsafe_wrapper(ctx_fn)) - } - }, - // We only use macros to check const generics (using static assertions). - Self::MacroCall(_name, _args) => false, - // Materialising uninitialised values is always unsafe, and we avoid it in safe - // functions. - Self::SvUndef => panic!("Refusing to wrap unsafe SvUndef in safe function '{ctx_fn}'."), - // Variants that aren't tokenised. We shouldn't encounter these here. - Self::MatchKind(..) => { - unimplemented!("The unsafety of {self:?} cannot be determined in '{ctx_fn}'.") - } - Self::MatchSize(..) => { - unimplemented!("The unsafety of {self:?} cannot be determined in '{ctx_fn}'.") - } - } - } -} - -impl FromStr for Expression { - type Err = String; - - fn from_str(s: &str) -> Result { - lazy_static! { - static ref MACRO_RE: Regex = - Regex::new(r"^(?P[\w\d_]+)!\((?P.*?)\);?$").unwrap(); - } - - if s == "SvUndef" { - Ok(Expression::SvUndef) - } else if MACRO_RE.is_match(s) { - let c = MACRO_RE.captures(s).unwrap(); - let ex = c["ex"].to_string(); - let _: TokenStream = ex - .parse() - .map_err(|e| format!("could not parse macro call expression: {e:#?}"))?; - Ok(Expression::MacroCall(c["name"].to_string(), ex)) - } else { - let (s, id_type) = if let Some(varname) = s.strip_prefix('$') { - (varname, IdentifierType::Variable) - } else { - (s, IdentifierType::Symbol) - }; - let identifier = s.trim().parse()?; - Ok(Expression::Identifier(identifier, id_type)) - } - } -} - -impl From for Expression { - fn from(fn_call: FnCall) -> Self { - Expression::FnCall(fn_call) - } -} - -impl From for Expression { - fn from(ws: WildString) -> Self { - Expression::Identifier(ws, IdentifierType::Symbol) - } -} - -impl From<&Argument> for Expression { - fn from(a: &Argument) -> Self { - Expression::Identifier(a.name.to_owned(), IdentifierType::Variable) - } -} - -impl TryFrom<&StaticDefinition> for Expression { - type Error = String; - - fn try_from(sd: &StaticDefinition) -> Result { - match sd { - StaticDefinition::Constant(imm) => Ok(imm.into()), - StaticDefinition::Generic(t) => t.parse(), - } - } -} - -impl fmt::Display for Expression { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Identifier(identifier, kind) => { - write!( - f, - "{}{identifier}", - matches!(kind, IdentifierType::Variable) - .then_some("$") - .unwrap_or_default() - ) - } - Self::MacroCall(name, expression) => { - write!(f, "{name}!({expression})") - } - _ => Err(fmt::Error), - } - } -} - -impl ToTokens for Expression { - fn to_tokens(&self, tokens: &mut TokenStream) { - match self { - Self::Let(LetVariant::Basic(var_name, exp)) => { - let var_ident = format_ident!("{}", var_name.to_string()); - tokens.append_all(quote! { let #var_ident = #exp }) - } - Self::Let(LetVariant::WithType(var_name, ty, exp)) => { - let var_ident = format_ident!("{}", var_name.to_string()); - tokens.append_all(quote! { let #var_ident: #ty = #exp }) - } - Self::Assign(var_name, exp) => { - /* If we are dereferencing a variable to assign a value \ - * the 'format_ident!' macro does not like the asterix */ - let var_name_str: &str; - - if let Some(ch) = var_name.chars().nth(0) { - /* Manually append the asterix and split out the rest of - * the variable name */ - if ch == '*' { - tokens.append(Punct::new('*', Spacing::Alone)); - var_name_str = &var_name[1..var_name.len()]; - } else { - var_name_str = var_name.as_str(); - } - } else { - /* Should not be reached as you cannot have a variable - * without a name */ - panic!("Invalid variable name, must be at least one character") - } - - let var_ident = format_ident!("{}", var_name_str); - tokens.append_all(quote! { #var_ident = #exp }) - } - Self::MacroCall(name, ex) => { - let name = format_ident!("{name}"); - let ex: TokenStream = ex.parse().unwrap(); - tokens.append_all(quote! { #name!(#ex) }) - } - Self::FnCall(fn_call) => fn_call.to_tokens(tokens), - Self::MethodCall(exp, fn_name, args) => { - let fn_ident = format_ident!("{}", fn_name); - tokens.append_all(quote! { #exp.#fn_ident(#(#args),*) }) - } - Self::Identifier(identifier, _) => { - assert!( - !identifier.has_wildcards(), - "expression {self:#?} was not built before calling to_tokens" - ); - identifier - .to_string() - .parse::() - .expect(format!("invalid syntax: {:?}", self).as_str()) - .to_tokens(tokens); - } - Self::IntConstant(n) => tokens.append(Literal::i32_unsuffixed(*n)), - Self::FloatConstant(n) => tokens.append(Literal::f32_unsuffixed(*n)), - Self::BoolConstant(true) => tokens.append(format_ident!("true")), - Self::BoolConstant(false) => tokens.append(format_ident!("false")), - Self::Array(vec) => tokens.append_all(quote! { [ #(#vec),* ] }), - Self::LLVMLink(link) => link.to_tokens(tokens), - Self::CastAs(ex, ty) => { - let ty: TokenStream = ty.parse().expect("invalid syntax"); - tokens.append_all(quote! { #ex as #ty }) - } - Self::SvUndef => tokens.append_all(quote! { simd_reinterpret(()) }), - Self::Multiply(lhs, rhs) => tokens.append_all(quote! { #lhs * #rhs }), - Self::Xor(lhs, rhs) => tokens.append_all(quote! { #lhs ^ #rhs }), - Self::Type(ty) => ty.to_tokens(tokens), - _ => unreachable!("{self:?} cannot be converted to tokens."), - } - } -} - -impl Serialize for Expression { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - Self::IntConstant(v) => serializer.serialize_i32(*v), - Self::FloatConstant(v) => serializer.serialize_f32(*v), - Self::BoolConstant(v) => serializer.serialize_bool(*v), - Self::Identifier(..) => serializer.serialize_str(&self.to_string()), - Self::MacroCall(..) => serializer.serialize_str(&self.to_string()), - _ => Expression::serialize(self, serializer), - } - } -} - -impl<'de> Deserialize<'de> for Expression { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct CustomExpressionVisitor; - - impl<'de> Visitor<'de> for CustomExpressionVisitor { - type Value = Expression; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("integer, float, boolean, string or map") - } - - fn visit_bool(self, v: bool) -> Result - where - E: de::Error, - { - Ok(Expression::BoolConstant(v)) - } - - fn visit_i64(self, v: i64) -> Result - where - E: de::Error, - { - Ok(Expression::IntConstant(v as i32)) - } - - fn visit_u64(self, v: u64) -> Result - where - E: de::Error, - { - Ok(Expression::IntConstant(v as i32)) - } - - fn visit_f64(self, v: f64) -> Result - where - E: de::Error, - { - Ok(Expression::FloatConstant(v as f32)) - } - - fn visit_str(self, value: &str) -> Result - where - E: de::Error, - { - FromStr::from_str(value).map_err(de::Error::custom) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: de::SeqAccess<'de>, - { - let arr = std::iter::from_fn(|| seq.next_element::().transpose()) - .try_collect()?; - Ok(Expression::Array(arr)) - } - - fn visit_map(self, map: M) -> Result - where - M: MapAccess<'de>, - { - // `MapAccessDeserializer` is a wrapper that turns a `MapAccess` - // into a `Deserializer`, allowing it to be used as the input to T's - // `Deserialize` implementation. T then deserializes itself using - // the entries from the map visitor. - Expression::deserialize(de::value::MapAccessDeserializer::new(map)) - } - } - - deserializer.deserialize_any(CustomExpressionVisitor) - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/fn_suffix.rs b/library/stdarch/crates/stdarch-gen2/src/fn_suffix.rs deleted file mode 100644 index 9f7827776e0..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/fn_suffix.rs +++ /dev/null @@ -1,308 +0,0 @@ -use std::fmt::{self}; - -/* This file is acting as a bridge between the old neon types and how they - * have a fairly complex way of picking suffixes and the new world. If possible - * it would be good to clean this up. At least it is self contained and the - * logic simple */ -use crate::typekinds::{BaseType, BaseTypeKind, TypeKind, VectorType}; -use serde::{Deserialize, Serialize}; - -use std::str::FromStr; - -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Deserialize, Serialize)] -pub enum SuffixKind { - Normal, - Base, - NoQ, - NSuffix, - NoQNSuffix, - DupNox, - Dup, - /* Get the number of lanes or panic if there are not any Lanes */ - Lane, - Rot270, - Rot270Lane, - Rot270LaneQ, - Rot180, - Rot180Lane, - Rot180LaneQ, - Rot90, - Rot90Lane, - Rot90LaneQ, - /* Force the type to be unsigned */ - Unsigned, - Tuple, - NoX, - BaseByteSize, - LaneNoX, - LaneQNoX, -} - -pub fn type_to_size(str_type: &str) -> i32 { - match str_type { - "int8x8_t" | "int8x16_t" | "i8" | "s8" | "uint8x8_t" | "uint8x16_t" | "u8" - | "poly8x8_t" | "poly8x16_t" => 8, - "int16x4_t" | "int16x8_t" | "i16" | "s16" | "uint16x4_t" | "uint16x8_t" | "u16" - | "float16x4_t" | "float16x8_t" | "_f16" | "poly16x4_t" | "poly16x8_t" => 16, - "int32x2_t" | "int32x4_t" | "i32" | "s32" | "uint32x2_t" | "uint32x4_t" | "u32" - | "float32x2_t" | "float32x4_t" | "f32" => 32, - "int64x1_t" | "int64x2_t" | "i64" | "s64" | "uint64x1_t" | "uint64x2_t" | "u64" - | "float64x1_t" | "float64x2_t" | "f64" | "poly64x1_t" | "poly64x2_t" | "p64" => 64, - "p128" => 128, - _ => panic!("unknown type: {str_type}"), - } -} - -fn neon_get_base_and_char(ty: &VectorType) -> (u32, char, bool) { - let lanes = ty.lanes(); - match ty.base_type() { - BaseType::Sized(BaseTypeKind::Float, size) => (*size, 'f', *size * lanes == 128), - BaseType::Sized(BaseTypeKind::Int, size) => (*size, 's', *size * lanes == 128), - BaseType::Sized(BaseTypeKind::UInt, size) => (*size, 'u', *size * lanes == 128), - BaseType::Sized(BaseTypeKind::Poly, size) => (*size, 'p', *size * lanes == 128), - _ => panic!("Unhandled {:?}", ty), - } -} - -/* @TODO - * for the chained enum types we can safely delete them as we can index the - * types array */ -pub fn make_neon_suffix(type_kind: TypeKind, suffix_kind: SuffixKind) -> String { - match type_kind { - TypeKind::Vector(ty) => { - let tuple_size = ty.tuple_size().map_or(0, |t| t.to_int()); - let (base_size, prefix_char, requires_q) = neon_get_base_and_char(&ty); - let lanes = ty.lanes(); - match suffix_kind { - SuffixKind::Normal => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - if tuple_size > 0 { - str_suffix.push_str("_x"); - str_suffix.push_str(tuple_size.to_string().as_str()); - } - return str_suffix; - } - SuffixKind::NSuffix => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push_str("_n_"); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - return str_suffix; - } - - SuffixKind::NoQ => format!("_{}{}", prefix_char, base_size), - SuffixKind::NoQNSuffix => format!("_n{}{}", prefix_char, base_size), - - SuffixKind::Unsigned => { - let t = type_kind.to_string(); - if t.starts_with("u") { - return t; - } - return format!("u{}", t); - } - SuffixKind::Lane => { - if lanes == 0 { - panic!("type {} has no lanes!", type_kind.to_string()) - } else { - format!("{}", lanes) - } - } - SuffixKind::Tuple => { - if tuple_size == 0 { - panic!("type {} has no lanes!", type_kind.to_string()) - } else { - format!("{}", tuple_size) - } - } - SuffixKind::Base => base_size.to_string(), - SuffixKind::NoX => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - return str_suffix; - } - SuffixKind::Dup => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push_str("dup_"); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - if tuple_size > 0 { - str_suffix.push_str("_x"); - str_suffix.push_str(tuple_size.to_string().as_str()); - } - return str_suffix; - } - SuffixKind::DupNox => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push_str("dup_"); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - return str_suffix; - } - SuffixKind::LaneNoX => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push_str("lane_"); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - return str_suffix; - } - SuffixKind::LaneQNoX => { - let mut str_suffix: String = String::new(); - if requires_q { - str_suffix.push('q'); - } - str_suffix.push('_'); - str_suffix.push_str("laneq_"); - str_suffix.push(prefix_char); - str_suffix.push_str(base_size.to_string().as_str()); - return str_suffix; - } - SuffixKind::Rot270 => { - if requires_q { - return format!("q_rot270_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot270_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot270Lane => { - if requires_q { - return format!("q_rot270_lane_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot270_lane_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot270LaneQ => { - if requires_q { - return format!("q_rot270_laneq_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot270_laneq_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot180 => { - if requires_q { - return format!("q_rot180_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot180_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot180Lane => { - if requires_q { - return format!("q_rot180_lane_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot180_lane_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot180LaneQ => { - if requires_q { - return format!("q_rot180_laneq_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot180_laneq_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot90 => { - if requires_q { - return format!("q_rot90_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot90_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot90Lane => { - if requires_q { - return format!("q_rot90_lane_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot90_lane_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::Rot90LaneQ => { - if requires_q { - return format!("q_rot90_laneq_{}{}", prefix_char, base_size.to_string()); - } - return format!("_rot90_laneq_{}{}", prefix_char, base_size.to_string()); - } - SuffixKind::BaseByteSize => format!("{}", base_size / 8), - } - } - _ => panic!("Cannot only make neon vector types suffixed"), - } -} - -impl FromStr for SuffixKind { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "no" => Ok(SuffixKind::Normal), - "noq" => Ok(SuffixKind::NoQ), - "N" => Ok(SuffixKind::NSuffix), - "noq_N" => Ok(SuffixKind::NoQNSuffix), - "dup_nox" => Ok(SuffixKind::DupNox), - "dup" => Ok(SuffixKind::Dup), - "lane" => Ok(SuffixKind::Lane), - "base" => Ok(SuffixKind::Base), - "tuple" => Ok(SuffixKind::Tuple), - "rot270" => Ok(SuffixKind::Rot270), - "rot270_lane" => Ok(SuffixKind::Rot270Lane), - "rot270_laneq" => Ok(SuffixKind::Rot270LaneQ), - "rot90" => Ok(SuffixKind::Rot90), - "rot90_lane" => Ok(SuffixKind::Rot90Lane), - "rot90_laneq" => Ok(SuffixKind::Rot90LaneQ), - "rot180" => Ok(SuffixKind::Rot180), - "rot180_lane" => Ok(SuffixKind::Rot180LaneQ), - "rot180_laneq" => Ok(SuffixKind::Rot180LaneQ), - "u" => Ok(SuffixKind::Unsigned), - "nox" => Ok(SuffixKind::NoX), - "base_byte_size" => Ok(SuffixKind::BaseByteSize), - "lane_nox" => Ok(SuffixKind::LaneNoX), - "laneq_nox" => Ok(SuffixKind::LaneQNoX), - _ => Err(format!("unknown suffix type: {}", s)), - } - } -} - -impl fmt::Display for SuffixKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - SuffixKind::Normal => write!(f, "normal"), - SuffixKind::NoQ => write!(f, "NoQ"), - SuffixKind::NSuffix => write!(f, "NSuffix"), - SuffixKind::NoQNSuffix => write!(f, "NoQNSuffix"), - SuffixKind::DupNox => write!(f, "DupNox"), - SuffixKind::Dup => write!(f, "Dup",), - SuffixKind::Lane => write!(f, "Lane"), - SuffixKind::LaneNoX => write!(f, "LaneNoX"), - SuffixKind::LaneQNoX => write!(f, "LaneQNoX"), - SuffixKind::Base => write!(f, "Base"), - SuffixKind::Rot270 => write!(f, "Rot270",), - SuffixKind::Rot270Lane => write!(f, "Rot270Lane"), - SuffixKind::Rot270LaneQ => write!(f, "Rot270LaneQ"), - SuffixKind::Rot90 => write!(f, "Rot90",), - SuffixKind::Rot90Lane => write!(f, "Rot90Lane"), - SuffixKind::Rot90LaneQ => write!(f, "Rot90LaneQ"), - SuffixKind::Rot180 => write!(f, "Rot180",), - SuffixKind::Rot180Lane => write!(f, "Rot180Lane"), - SuffixKind::Rot180LaneQ => write!(f, "Rot180LaneQ"), - SuffixKind::Unsigned => write!(f, "Unsigned"), - SuffixKind::Tuple => write!(f, "Tuple"), - SuffixKind::NoX => write!(f, "NoX"), - SuffixKind::BaseByteSize => write!(f, "BaseByteSize"), - } - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/input.rs b/library/stdarch/crates/stdarch-gen2/src/input.rs deleted file mode 100644 index bb2414adec2..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/input.rs +++ /dev/null @@ -1,432 +0,0 @@ -use itertools::Itertools; -use serde::{de, Deserialize, Deserializer, Serialize}; - -use crate::{ - context::{self, GlobalContext}, - intrinsic::Intrinsic, - predicate_forms::{PredicateForm, PredicationMask, PredicationMethods}, - typekinds::TypeKind, - wildstring::WildString, -}; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum InputType { - /// PredicateForm variant argument - #[serde(skip)] // Predicate forms have their own dedicated deserialization field. Skip. - PredicateForm(PredicateForm), - /// Operand from which to generate an N variant - #[serde(skip)] - NVariantOp(Option), - /// TypeKind variant argument - Type(TypeKind), -} - -impl InputType { - /// Optionally unwraps as a PredicateForm. - pub fn predicate_form(&self) -> Option<&PredicateForm> { - match self { - InputType::PredicateForm(pf) => Some(pf), - _ => None, - } - } - - /// Optionally unwraps as a mutable PredicateForm - pub fn predicate_form_mut(&mut self) -> Option<&mut PredicateForm> { - match self { - InputType::PredicateForm(pf) => Some(pf), - _ => None, - } - } - - /// Optionally unwraps as a TypeKind. - pub fn typekind(&self) -> Option<&TypeKind> { - match self { - InputType::Type(ty) => Some(ty), - _ => None, - } - } - - /// Optionally unwraps as a NVariantOp - pub fn n_variant_op(&self) -> Option<&WildString> { - match self { - InputType::NVariantOp(Some(op)) => Some(op), - _ => None, - } - } -} - -impl PartialOrd for InputType { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for InputType { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use std::cmp::Ordering::*; - - match (self, other) { - (InputType::PredicateForm(pf1), InputType::PredicateForm(pf2)) => pf1.cmp(pf2), - (InputType::Type(ty1), InputType::Type(ty2)) => ty1.cmp(ty2), - - (InputType::NVariantOp(None), InputType::NVariantOp(Some(..))) => Less, - (InputType::NVariantOp(Some(..)), InputType::NVariantOp(None)) => Greater, - (InputType::NVariantOp(_), InputType::NVariantOp(_)) => Equal, - - (InputType::Type(..), InputType::PredicateForm(..)) => Less, - (InputType::PredicateForm(..), InputType::Type(..)) => Greater, - - (InputType::Type(..), InputType::NVariantOp(..)) => Less, - (InputType::NVariantOp(..), InputType::Type(..)) => Greater, - - (InputType::PredicateForm(..), InputType::NVariantOp(..)) => Less, - (InputType::NVariantOp(..), InputType::PredicateForm(..)) => Greater, - } - } -} - -mod many_or_one { - use serde::{de::Deserializer, ser::Serializer, Deserialize, Serialize}; - - pub fn serialize(vec: &Vec, serializer: S) -> Result - where - T: Serialize, - S: Serializer, - { - if vec.len() == 1 { - vec.first().unwrap().serialize(serializer) - } else { - vec.serialize(serializer) - } - } - - pub fn deserialize<'de, T, D>(deserializer: D) -> Result, D::Error> - where - T: Deserialize<'de>, - D: Deserializer<'de>, - { - #[derive(Debug, Clone, Serialize, Deserialize)] - #[serde(untagged)] - enum ManyOrOne { - Many(Vec), - One(T), - } - - match ManyOrOne::deserialize(deserializer)? { - ManyOrOne::Many(vec) => Ok(vec), - ManyOrOne::One(val) => Ok(vec![val]), - } - } -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct InputSet(#[serde(with = "many_or_one")] Vec); - -impl InputSet { - pub fn get(&self, idx: usize) -> Option<&InputType> { - self.0.get(idx) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn iter(&self) -> impl Iterator + '_ { - self.0.iter() - } - - pub fn iter_mut(&mut self) -> impl Iterator + '_ { - self.0.iter_mut() - } - - pub fn into_iter(self) -> impl Iterator + Clone { - self.0.into_iter() - } - - pub fn types_len(&self) -> usize { - self.iter().filter_map(|arg| arg.typekind()).count() - } - - pub fn typekind(&self, idx: Option) -> Option { - let types_len = self.types_len(); - self.get(idx.unwrap_or(0)).and_then(move |arg: &InputType| { - if (idx.is_none() && types_len != 1) || (idx.is_some() && types_len == 1) { - None - } else { - arg.typekind().cloned() - } - }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct InputSetEntry(#[serde(with = "many_or_one")] Vec); - -impl InputSetEntry { - pub fn new(input: Vec) -> Self { - Self(input) - } - - pub fn get(&self, idx: usize) -> Option<&InputSet> { - self.0.get(idx) - } -} - -fn validate_types<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let v: Vec = Vec::deserialize(deserializer)?; - - let mut it = v.iter(); - if let Some(first) = it.next() { - it.try_fold(first, |last, cur| { - if last.0.len() == cur.0.len() { - Ok(cur) - } else { - Err("the length of the InputSets and the product lists must match".to_string()) - } - }) - .map_err(de::Error::custom)?; - } - - Ok(v) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct IntrinsicInput { - #[serde(default)] - #[serde(deserialize_with = "validate_types")] - pub types: Vec, - - #[serde(flatten)] - pub predication_methods: PredicationMethods, - - /// Generates a _n variant where the specified operand is a primitive type - /// that requires conversion to an SVE one. The `{_n}` wildcard is required - /// in the intrinsic's name, otherwise an error will be thrown. - #[serde(default)] - pub n_variant_op: WildString, -} - -impl IntrinsicInput { - /// Extracts all the possible variants as an iterator. - pub fn variants( - &self, - intrinsic: &Intrinsic, - ) -> context::Result + '_> { - let mut top_product = vec![]; - - if !self.types.is_empty() { - top_product.push( - self.types - .iter() - .flat_map(|ty_in| { - ty_in - .0 - .iter() - .map(|v| v.clone().into_iter()) - .multi_cartesian_product() - }) - .collect_vec(), - ) - } - - if let Ok(mask) = PredicationMask::try_from(&intrinsic.signature.name) { - top_product.push( - PredicateForm::compile_list(&mask, &self.predication_methods)? - .into_iter() - .map(|pf| vec![InputType::PredicateForm(pf)]) - .collect_vec(), - ) - } - - if !self.n_variant_op.is_empty() { - top_product.push(vec![ - vec![InputType::NVariantOp(None)], - vec![InputType::NVariantOp(Some(self.n_variant_op.to_owned()))], - ]) - } - - let it = top_product - .into_iter() - .map(|v| v.into_iter()) - .multi_cartesian_product() - .map(|set| InputSet(set.into_iter().flatten().collect_vec())); - Ok(it) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GeneratorInput { - #[serde(flatten)] - pub ctx: GlobalContext, - pub intrinsics: Vec, -} - -#[cfg(test)] -mod tests { - use crate::{ - input::*, - predicate_forms::{DontCareMethod, ZeroingMethod}, - }; - - #[test] - fn test_empty() { - let str = r#"types: []"#; - let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); - let mut variants = input.variants(&Intrinsic::default()).unwrap().into_iter(); - assert_eq!(variants.next(), None); - } - - #[test] - fn test_product() { - let str = r#"types: -- [f64, f32] -- [i64, [f64, f32]] -"#; - let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); - let mut intrinsic = Intrinsic::default(); - intrinsic.signature.name = "test_intrinsic{_mx}".parse().unwrap(); - let mut variants = input.variants(&intrinsic).unwrap().into_iter(); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("f64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::PredicateForm(PredicateForm::Merging), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("f64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::Type("f64".parse().unwrap()), - InputType::PredicateForm(PredicateForm::Merging), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::Type("f64".parse().unwrap()), - InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::PredicateForm(PredicateForm::Merging), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsMerging)), - ])), - ); - assert_eq!(variants.next(), None); - } - - #[test] - fn test_n_variant() { - let str = r#"types: -- [f64, f32] -n_variant_op: op2 -"#; - let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); - let mut variants = input.variants(&Intrinsic::default()).unwrap().into_iter(); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("f64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::NVariantOp(None), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("f64".parse().unwrap()), - InputType::Type("f32".parse().unwrap()), - InputType::NVariantOp(Some("op2".parse().unwrap())), - ])) - ); - assert_eq!(variants.next(), None) - } - - #[test] - fn test_invalid_length() { - let str = r#"types: [i32, [[u64], [u32]]]"#; - serde_yaml::from_str::(str).expect_err("failure expected"); - } - - #[test] - fn test_invalid_predication() { - let str = "types: []"; - let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); - let mut intrinsic = Intrinsic::default(); - intrinsic.signature.name = "test_intrinsic{_mxz}".parse().unwrap(); - input - .variants(&intrinsic) - .map(|v| v.collect_vec()) - .expect_err("failure expected"); - } - - #[test] - fn test_invalid_predication_mask() { - "test_intrinsic{_mxy}" - .parse::() - .expect_err("failure expected"); - "test_intrinsic{_}" - .parse::() - .expect_err("failure expected"); - } - - #[test] - fn test_zeroing_predication() { - let str = r#"types: [i64] -zeroing_method: { drop: inactive }"#; - let input: IntrinsicInput = serde_yaml::from_str(str).expect("failed to parse"); - let mut intrinsic = Intrinsic::default(); - intrinsic.signature.name = "test_intrinsic{_mxz}".parse().unwrap(); - let mut variants = input.variants(&intrinsic).unwrap(); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::PredicateForm(PredicateForm::Merging), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::PredicateForm(PredicateForm::DontCare(DontCareMethod::AsZeroing)), - ])) - ); - assert_eq!( - variants.next(), - Some(InputSet(vec![ - InputType::Type("i64".parse().unwrap()), - InputType::PredicateForm(PredicateForm::Zeroing(ZeroingMethod::Drop { - drop: "inactive".parse().unwrap() - })), - ])) - ); - assert_eq!(variants.next(), None) - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/intrinsic.rs b/library/stdarch/crates/stdarch-gen2/src/intrinsic.rs deleted file mode 100644 index 4a973691b17..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/intrinsic.rs +++ /dev/null @@ -1,1624 +0,0 @@ -use itertools::Itertools; -use proc_macro2::{Punct, Spacing, TokenStream}; -use quote::{format_ident, quote, ToTokens, TokenStreamExt}; -use serde::{Deserialize, Serialize}; -use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::collections::{HashMap, HashSet}; -use std::fmt::{self}; -use std::num::ParseIntError; -use std::ops::RangeInclusive; -use std::str::FromStr; - -use crate::assert_instr::InstructionAssertionsForBaseType; -use crate::context::{GlobalContext, GroupContext}; -use crate::input::{InputSet, InputSetEntry}; -use crate::predicate_forms::{DontCareMethod, PredicateForm, PredicationMask, ZeroingMethod}; -use crate::{ - assert_instr::InstructionAssertionMethod, - context::{self, ArchitectureSettings, Context, LocalContext, VariableType}, - expression::{Expression, FnCall, IdentifierType}, - fn_suffix::{type_to_size, SuffixKind}, - input::IntrinsicInput, - matching::{KindMatchable, SizeMatchable}, - typekinds::*, - wildcards::Wildcard, - wildstring::WildString, -}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum SubstitutionType { - MatchSize(SizeMatchable), - MatchKind(KindMatchable), -} - -impl SubstitutionType { - pub fn get(&mut self, ctx: &LocalContext) -> context::Result { - match self { - Self::MatchSize(smws) => { - smws.perform_match(ctx)?; - Ok(smws.as_ref().clone()) - } - Self::MatchKind(kmws) => { - kmws.perform_match(ctx)?; - Ok(kmws.as_ref().clone()) - } - } - } -} - -/// Mutability level -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum AccessLevel { - /// Immutable - R, - /// Mutable - RW, -} - -/// Function signature argument. -/// -/// Prepend the `mut` keyword for a mutable argument. Separate argument name -/// and type with a semicolon `:`. Usage examples: -/// - Mutable argument: `mut arg1: *u64` -/// - Immutable argument: `arg2: u32` -#[derive(Debug, Clone, SerializeDisplay, DeserializeFromStr)] -pub struct Argument { - /// Argument name - pub name: WildString, - /// Mutability level - pub rw: AccessLevel, - /// Argument type - pub kind: TypeKind, -} - -impl Argument { - pub fn populate_variables(&self, vars: &mut HashMap) { - vars.insert( - self.name.to_string(), - (self.kind.clone(), VariableType::Argument), - ); - } -} - -impl FromStr for Argument { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut it = s.splitn(2, ':').map(::trim); - if let Some(mut lhs) = it.next().map(|s| s.split_whitespace()) { - let lhs_len = lhs.clone().count(); - match (lhs_len, lhs.next(), it.next()) { - (2, Some("mut"), Some(kind)) => Ok(Argument { - name: lhs.next().unwrap().parse()?, - rw: AccessLevel::RW, - kind: kind.parse()?, - }), - (2, Some(ident), _) => Err(format!("invalid {ident:#?} keyword")), - (1, Some(name), Some(kind)) => Ok(Argument { - name: name.parse()?, - rw: AccessLevel::R, - kind: kind.parse()?, - }), - _ => Err(format!("invalid argument `{s}` provided")), - } - } else { - Err(format!("invalid argument `{s}` provided")) - } - } -} - -impl fmt::Display for Argument { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if let AccessLevel::RW = &self.rw { - write!(f, "mut ")?; - } - - write!(f, "{}: {}", self.name, self.kind) - } -} - -impl ToTokens for Argument { - fn to_tokens(&self, tokens: &mut TokenStream) { - if let AccessLevel::RW = &self.rw { - tokens.append(format_ident!("mut")) - } - - let (name, kind) = (format_ident!("{}", self.name.to_string()), &self.kind); - tokens.append_all(quote! { #name: #kind }) - } -} - -/// Static definition part of the signature. It may evaluate to a constant -/// expression with e.g. `const imm: u64`, or a generic `T: Into`. -#[derive(Debug, Clone, SerializeDisplay, DeserializeFromStr)] -pub enum StaticDefinition { - /// Constant expression - Constant(Argument), - /// Generic type - Generic(String), -} - -impl StaticDefinition { - pub fn as_variable(&self) -> Option<(String, (TypeKind, VariableType))> { - match self { - StaticDefinition::Constant(arg) => Some(( - arg.name.to_string(), - (arg.kind.clone(), VariableType::Argument), - )), - StaticDefinition::Generic(..) => None, - } - } -} - -impl FromStr for StaticDefinition { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.trim() { - s if s.starts_with("const ") => Ok(StaticDefinition::Constant(s[6..].trim().parse()?)), - s => Ok(StaticDefinition::Generic(s.to_string())), - } - } -} - -impl fmt::Display for StaticDefinition { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - StaticDefinition::Constant(arg) => write!(f, "const {arg}"), - StaticDefinition::Generic(generic) => write!(f, "{generic}"), - } - } -} - -impl ToTokens for StaticDefinition { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.append_all(match self { - StaticDefinition::Constant(arg) => quote! { const #arg }, - StaticDefinition::Generic(generic) => { - let generic: TokenStream = generic.parse().expect("invalid Rust code"); - quote! { #generic } - } - }) - } -} - -/// Function constraints -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum Constraint { - /// Asserts that the given variable equals to any of the given integer values - AnyI32 { - variable: String, - any_values: Vec, - }, - /// WildString version of RangeI32. If the string values given for the range - /// are valid, this gets built into a RangeI32. - RangeWildstring { - variable: String, - range: (WildString, WildString), - }, - /// Asserts that the given variable's value falls in the specified range - RangeI32 { - variable: String, - range: SizeMatchable>, - }, - /// Asserts that the number of elements/lanes does not exceed the 2048-bit SVE constraint - SVEMaxElems { - variable: String, - sve_max_elems_type: TypeKind, - }, - /// Asserts that the number of elements/lanes does not exceed the 128-bit register constraint - VecMaxElems { - variable: String, - vec_max_elems_type: TypeKind, - }, -} - -impl Constraint { - fn variable(&self) -> &str { - match self { - Constraint::AnyI32 { variable, .. } - | Constraint::RangeWildstring { variable, .. } - | Constraint::RangeI32 { variable, .. } - | Constraint::SVEMaxElems { variable, .. } - | Constraint::VecMaxElems { variable, .. } => variable, - } - } - pub fn build(&mut self, ctx: &Context) -> context::Result { - if let Self::RangeWildstring { - variable, - range: (min, max), - } = self - { - min.build_acle(ctx.local)?; - max.build_acle(ctx.local)?; - let min = min.to_string(); - let max = max.to_string(); - let min: i32 = min - .parse() - .map_err(|_| format!("the minimum value `{min}` is not a valid number"))?; - let max: i32 = max - .parse() - .or_else(|_| Ok(type_to_size(max.as_str()))) - .map_err(|_: ParseIntError| { - format!("the maximum value `{max}` is not a valid number") - })?; - *self = Self::RangeI32 { - variable: variable.to_owned(), - range: SizeMatchable::Matched(RangeInclusive::new(min, max)), - } - } - - if let Self::SVEMaxElems { - sve_max_elems_type: ty, - .. - } - | Self::VecMaxElems { - vec_max_elems_type: ty, - .. - } = self - { - if let Some(w) = ty.wildcard() { - ty.populate_wildcard(ctx.local.provide_type_wildcard(w)?)?; - } - } - - if let Self::RangeI32 { range, .. } = self { - range.perform_match(ctx.local)?; - } - - let variable = self.variable(); - ctx.local - .variables - .contains_key(variable) - .then_some(()) - .ok_or_else(|| format!("cannot build constraint, could not find variable {variable}")) - } -} - -/// Function signature -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct Signature { - /// Function name - pub name: WildString, - /// List of function arguments, leave unset or empty for no arguments - pub arguments: Vec, - /// Function return type, leave unset for void - pub return_type: Option, - - /// For some neon intrinsics we want to modify the suffix of the function name - pub suffix_type: Option, - - /// List of static definitions, leave unset of empty if not required - #[serde(default)] - pub static_defs: Vec, - - /// **Internal use only.** - /// Condition for which the ultimate function is specific to predicates. - #[serde(skip)] - pub is_predicate_specific: bool, - - /// **Internal use only.** - /// Setting this property will trigger the signature builder to convert any `svbool*_t` to `svbool_t` in the input and output. - #[serde(skip)] - pub predicate_needs_conversion: bool, -} - -impl Signature { - pub fn drop_argument(&mut self, arg_name: &WildString) -> Result<(), String> { - if let Some(idx) = self - .arguments - .iter() - .position(|arg| arg.name.to_string() == arg_name.to_string()) - { - self.arguments.remove(idx); - Ok(()) - } else { - Err(format!("no argument {arg_name} found to drop")) - } - } - - pub fn build(&mut self, ctx: &LocalContext) -> context::Result { - if self.name_has_neon_suffix() { - self.name.build_neon_intrinsic_signature(ctx)?; - } else { - self.name.build_acle(ctx)?; - } - - if let Some(ref mut return_type) = self.return_type { - if let Some(w) = return_type.clone().wildcard() { - return_type.populate_wildcard(ctx.provide_type_wildcard(w)?)?; - } - } - - self.arguments - .iter_mut() - .try_for_each(|arg| arg.name.build_acle(ctx))?; - - self.arguments - .iter_mut() - .filter_map(|arg| { - arg.kind - .clone() - .wildcard() - .map(|w| (&mut arg.kind, w.clone())) - }) - .try_for_each(|(ty, w)| ty.populate_wildcard(ctx.provide_type_wildcard(&w)?)) - } - - pub fn fn_name(&self) -> WildString { - self.name.replace(['[', ']'], "") - } - - pub fn doc_name(&self) -> String { - self.name.to_string() - } - - fn name_has_neon_suffix(&self) -> bool { - for part in self.name.wildcards() { - let has_suffix = match part { - Wildcard::NEONType(_, _, suffix_type) => suffix_type.is_some(), - _ => false, - }; - - if has_suffix { - return true; - } - } - false - } -} - -impl ToTokens for Signature { - fn to_tokens(&self, tokens: &mut TokenStream) { - let name_ident = format_ident!("{}", self.fn_name().to_string()); - let arguments = self - .arguments - .clone() - .into_iter() - .map(|mut arg| { - if arg - .kind - .vector() - .map_or(false, |ty| ty.base_type().is_bool()) - && self.predicate_needs_conversion - { - arg.kind = TypeKind::Vector(VectorType::make_predicate_from_bitsize(8)) - } - arg - }) - .collect_vec(); - let static_defs = &self.static_defs; - tokens.append_all(quote! { fn #name_ident<#(#static_defs),*>(#(#arguments),*) }); - - if let Some(ref return_type) = self.return_type { - if return_type - .vector() - .map_or(false, |ty| ty.base_type().is_bool()) - && self.predicate_needs_conversion - { - tokens.append_all(quote! { -> svbool_t }) - } else { - tokens.append_all(quote! { -> #return_type }) - } - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LLVMLinkAttribute { - /// Either one architecture or a comma separated list of architectures with NO spaces - pub arch: String, - pub link: WildString, -} - -impl ToTokens for LLVMLinkAttribute { - fn to_tokens(&self, tokens: &mut TokenStream) { - let LLVMLinkAttribute { arch, link } = self; - let link = link.to_string(); - let archs: Vec<&str> = arch.split(',').collect(); - let arch_len = archs.len(); - - if arch_len == 1 { - tokens.append_all(quote! { - #[cfg_attr(target_arch = #arch, link_name = #link)] - }) - } else { - tokens.append(Punct::new('#', Spacing::Alone)); - tokens.append(Punct::new('[', Spacing::Alone)); - tokens.append_all(quote! { cfg_attr }); - tokens.append(Punct::new('(', Spacing::Alone)); - tokens.append_all(quote! { any }); - tokens.append(Punct::new('(', Spacing::Alone)); - let mut i = 0; - while i < arch_len { - let arch = archs[i].to_string(); - tokens.append_all(quote! { target_arch = #arch }); - if i + 1 != arch_len { - tokens.append(Punct::new(',', Spacing::Alone)); - } - i += 1; - } - tokens.append(Punct::new(')', Spacing::Alone)); - tokens.append(Punct::new(',', Spacing::Alone)); - tokens.append_all(quote! { link_name = #link }); - tokens.append(Punct::new(')', Spacing::Alone)); - tokens.append(Punct::new(']', Spacing::Alone)); - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LLVMLink { - /// LLVM link function name without namespace and types, - /// e.g. `st1` in `llvm.aarch64.sve.st1.nxv4i32` - pub name: WildString, - - /// LLVM link signature arguments, leave unset if it inherits from intrinsic's signature - pub arguments: Option>, - /// LLVM link signature return type, leave unset if it inherits from intrinsic's signature - pub return_type: Option, - - /// **This will be set automatically if not set** - /// Attribute LLVM links for the function. First element is the architecture it targets, - /// second element is the LLVM link itself. - pub links: Option>, - - /// **Internal use only. Do not set.** - /// Generated signature from these `arguments` and/or `return_type` if set, and the intrinsic's signature. - #[serde(skip)] - pub signature: Option>, -} - -impl LLVMLink { - pub fn resolve(&self, cfg: &ArchitectureSettings) -> String { - self.name - .starts_with("llvm") - .then(|| self.name.to_string()) - .unwrap_or_else(|| format!("{}.{}", cfg.llvm_link_prefix, self.name)) - } - - pub fn build_and_save(&mut self, ctx: &mut Context) -> context::Result { - self.build(ctx)?; - - // Save LLVM link to the group context - ctx.global.arch_cfgs.iter().for_each(|cfg| { - ctx.group - .links - .insert(self.resolve(cfg), ctx.local.input.clone()); - }); - - Ok(()) - } - - pub fn build(&mut self, ctx: &mut Context) -> context::Result { - let mut sig_name = ctx.local.signature.name.clone(); - sig_name.prepend_str("_"); - - let mut sig = Signature { - name: sig_name, - arguments: self - .arguments - .clone() - .unwrap_or_else(|| ctx.local.signature.arguments.clone()), - return_type: self - .return_type - .clone() - .or_else(|| ctx.local.signature.return_type.clone()), - suffix_type: None, - static_defs: vec![], - is_predicate_specific: ctx.local.signature.is_predicate_specific, - predicate_needs_conversion: false, - }; - - sig.build(ctx.local)?; - self.name.build(ctx.local, TypeRepr::LLVMMachine)?; - - // Add link function name to context - ctx.local - .substitutions - .insert(Wildcard::LLVMLink, sig.fn_name().to_string()); - - self.signature = Some(Box::new(sig)); - - if let Some(ref mut links) = self.links { - links.iter_mut().for_each(|ele| { - ele.link - .build(&ctx.local, TypeRepr::LLVMMachine) - .expect("Failed to transform to LLVMMachine representation"); - }); - } else { - self.links = Some( - ctx.global - .arch_cfgs - .iter() - .map(|cfg| LLVMLinkAttribute { - arch: cfg.arch_name.to_owned(), - link: self.resolve(cfg).into(), - }) - .collect_vec(), - ); - } - - Ok(()) - } - - /// Alters all the unsigned types from the signature, as unsupported by LLVM. - pub fn sanitise_uints(&mut self) { - let transform = |tk: &mut TypeKind| { - if let Some(BaseType::Sized(BaseTypeKind::UInt, size)) = tk.base_type() { - *tk.base_type_mut().unwrap() = BaseType::Sized(BaseTypeKind::Int, *size) - } - }; - - if let Some(sig) = self.signature.as_mut() { - for arg in sig.arguments.iter_mut() { - transform(&mut arg.kind); - } - - sig.return_type.as_mut().map(transform); - } - } - - /// Make a function call to the LLVM link - pub fn make_fn_call(&self, intrinsic_sig: &Signature) -> context::Result { - let link_sig = self.signature.as_ref().ok_or_else(|| { - "cannot derive the LLVM link call, as it does not hold a valid function signature" - .to_string() - })?; - - if intrinsic_sig.arguments.len() != link_sig.arguments.len() { - return Err( - "cannot derive the LLVM link call, the number of arguments does not match" - .to_string(), - ); - } - - let call_args = intrinsic_sig - .arguments - .iter() - .zip(link_sig.arguments.iter()) - .map(|(intrinsic_arg, link_arg)| { - // Could also add a type check... - if intrinsic_arg.name == link_arg.name { - Ok(Expression::Identifier( - intrinsic_arg.name.to_owned(), - IdentifierType::Variable, - )) - } else { - Err("cannot derive the LLVM link call, the arguments do not match".to_string()) - } - }) - .try_collect()?; - - Ok(FnCall::new_expression(link_sig.fn_name().into(), call_args)) - } - - /// Given a FnCall, apply all the predicate and unsigned conversions as required. - pub fn apply_conversions_to_call( - &self, - mut fn_call: FnCall, - ctx: &LocalContext, - ) -> context::Result { - use BaseType::{Sized, Unsized}; - use BaseTypeKind::{Bool, UInt}; - use VariableType::Argument; - - let convert = - |method: &str, ex| Expression::MethodCall(Box::new(ex), method.to_string(), vec![]); - - fn_call.1 = fn_call - .1 - .into_iter() - .map(|arg| -> context::Result { - if let Expression::Identifier(ref var_name, IdentifierType::Variable) = arg { - let (kind, scope) = ctx - .variables - .get(&var_name.to_string()) - .ok_or_else(|| format!("invalid variable {var_name:?} being referenced"))?; - - match (scope, kind.base_type()) { - (Argument, Some(Sized(Bool, bitsize))) if *bitsize != 8 => { - Ok(convert("into", arg)) - } - (Argument, Some(Sized(UInt, _) | Unsized(UInt))) => { - Ok(convert("as_signed", arg)) - } - _ => Ok(arg), - } - } else { - Ok(arg) - } - }) - .try_collect()?; - - let return_type_requires_conversion = self - .signature - .as_ref() - .and_then(|sig| sig.return_type.as_ref()) - .and_then(|ty| { - if let Some(Sized(Bool, bitsize)) = ty.base_type() { - (*bitsize != 8).then_some(Bool) - } else if let Some(Sized(UInt, _) | Unsized(UInt)) = ty.base_type() { - Some(UInt) - } else { - None - } - }); - - let fn_call = Expression::FnCall(fn_call); - match return_type_requires_conversion { - Some(Bool) => Ok(convert("into", fn_call)), - Some(UInt) => Ok(convert("as_unsigned", fn_call)), - _ => Ok(fn_call), - } - } -} - -impl ToTokens for LLVMLink { - fn to_tokens(&self, tokens: &mut TokenStream) { - assert!( - self.signature.is_some() && self.links.is_some(), - "expression {self:#?} was not built before calling to_tokens" - ); - - let signature = self.signature.as_ref().unwrap(); - let links = self.links.as_ref().unwrap(); - tokens.append_all(quote! { - extern "unadjusted" { - #(#links)* - #signature; - } - }) - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FunctionVisibility { - #[default] - Public, - Private, -} - -/// Whether to generate a load/store test, and which typeset index -/// represents the data type of the load/store target address -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Test { - #[default] - #[serde(skip)] - None, // Covered by `intrinsic-test` - Load(usize), - Store(usize), -} - -impl Test { - pub fn get_typeset_index(&self) -> Option { - match *self { - Test::Load(n) => Some(n), - Test::Store(n) => Some(n), - _ => None, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Safety { - Safe, - Unsafe(Vec), -} - -impl Safety { - /// Return `Ok(Safety::Safe)` if safety appears reasonable for the given `intrinsic`'s name and - /// prototype. Otherwise, return `Err()` with a suitable diagnostic. - fn safe_checked(intrinsic: &Intrinsic) -> Result { - let name = intrinsic.signature.doc_name(); - if name.starts_with("sv") { - let handles_pointers = intrinsic - .signature - .arguments - .iter() - .any(|arg| matches!(arg.kind, TypeKind::Pointer(..))); - if name.starts_with("svld") - || name.starts_with("svst") - || name.starts_with("svprf") - || name.starts_with("svundef") - || handles_pointers - { - let doc = intrinsic.doc.as_ref().map(|s| s.to_string()); - let doc = doc.as_deref().unwrap_or("..."); - Err(format!( - "`{name}` has no safety specification, but it looks like it should be unsafe. \ - Consider specifying (un)safety explicitly: - - - name: {name} - doc: {doc} - safety: - unsafe: - - ... - ... -" - )) - } else { - Ok(Self::Safe) - } - } else { - Err(format!( - "Safety::safe_checked() for non-SVE intrinsic: {name}" - )) - } - } - - fn is_safe(&self) -> bool { - match self { - Self::Safe => true, - Self::Unsafe(..) => false, - } - } - - fn is_unsafe(&self) -> bool { - !self.is_safe() - } - - fn has_doc_comments(&self) -> bool { - match self { - Self::Safe => false, - Self::Unsafe(v) => !v.is_empty(), - } - } - - fn doc_comments(&self) -> &[UnsafetyComment] { - match self { - Self::Safe => &[], - Self::Unsafe(v) => v.as_slice(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum UnsafetyComment { - Custom(String), - Uninitialized, - PointerOffset(GovernedBy), - PointerOffsetVnum(GovernedBy), - Dereference(GovernedBy), - UnpredictableOnFault, - NonTemporal, - Neon, - NoProvenance(String), -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum GovernedBy { - #[default] - Predicated, - PredicatedNonFaulting, - PredicatedFirstFaulting, -} - -impl fmt::Display for GovernedBy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Predicated => write!(f, " (governed by `pg`)"), - Self::PredicatedNonFaulting => write!( - f, - " (governed by `pg`, the first-fault register (`FFR`) \ - and non-faulting behaviour)" - ), - Self::PredicatedFirstFaulting => write!( - f, - " (governed by `pg`, the first-fault register (`FFR`) \ - and first-faulting behaviour)" - ), - } - } -} - -impl fmt::Display for UnsafetyComment { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Custom(s) => s.fmt(f), - Self::Neon => write!(f, "Neon instrinsic unsafe"), - Self::Uninitialized => write!( - f, - "This creates an uninitialized value, and may be unsound (like \ - [`core::mem::uninitialized`])." - ), - Self::PointerOffset(gov) => write!( - f, - "[`pointer::offset`](pointer#method.offset) safety constraints must \ - be met for the address calculation for each active element{gov}." - ), - Self::PointerOffsetVnum(gov) => write!( - f, - "[`pointer::offset`](pointer#method.offset) safety constraints must \ - be met for the address calculation for each active element{gov}. \ - In particular, note that `vnum` is scaled by the vector \ - length, `VL`, which is not known at compile time." - ), - Self::Dereference(gov) => write!( - f, - "This dereferences and accesses the calculated address for each \ - active element{gov}." - ), - Self::NonTemporal => write!( - f, - "Non-temporal accesses have special memory ordering rules, and \ - [explicit barriers may be required for some applications]\ - (https://developer.arm.com/documentation/den0024/a/Memory-Ordering/Barriers/Non-temporal-load-and-store-pair?lang=en)." - ), - Self::NoProvenance(arg) => write!( - f, - "Addresses passed in `{arg}` lack provenance, so this is similar to using a \ - `usize as ptr` cast (or [`core::ptr::from_exposed_addr`]) on each lane before \ - using it." - ), - Self::UnpredictableOnFault => write!( - f, - "Result lanes corresponding to inactive FFR lanes (either before or as a result \ - of this intrinsic) have \"CONSTRAINED UNPREDICTABLE\" values, irrespective of \ - predication. Refer to architectural documentation for details." - ), - } - } -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct Intrinsic { - #[serde(default)] - pub visibility: FunctionVisibility, - #[serde(default)] - pub doc: Option, - #[serde(flatten)] - pub signature: Signature, - /// Function sequential composition - pub compose: Vec, - /// Input to generate the intrinsic against. Leave empty if the intrinsic - /// does not have any variants. - /// Specific variants contain one InputSet - #[serde(flatten, default)] - pub input: IntrinsicInput, - #[serde(default)] - pub constraints: Vec, - /// Additional target features to add to the global settings - #[serde(default)] - pub target_features: Vec, - /// Should the intrinsic be `unsafe`? By default, the generator will try to guess from the - /// prototype, but it errs on the side of `unsafe`, and prints a warning in that case. - #[serde(default)] - pub safety: Option, - #[serde(default)] - pub substitutions: HashMap, - /// List of the only indices in a typeset that require conversion to signed - /// when deferring unsigned intrinsics to signed. (optional, default - /// behaviour is all unsigned types are converted to signed) - #[serde(default)] - pub defer_to_signed_only_indices: HashSet, - pub assert_instr: Option>, - /// Whether we should generate a test for this intrinsic - #[serde(default)] - pub test: Test, - /// Primary base type, used for instruction assertion. - #[serde(skip)] - pub base_type: Option, - /// Attributes for the function - pub attr: Option>, -} - -impl Intrinsic { - pub fn llvm_link(&self) -> Option<&LLVMLink> { - self.compose.iter().find_map(|ex| { - if let Expression::LLVMLink(llvm_link) = ex { - Some(llvm_link) - } else { - None - } - }) - } - - pub fn llvm_link_mut(&mut self) -> Option<&mut LLVMLink> { - self.compose.iter_mut().find_map(|ex| { - if let Expression::LLVMLink(llvm_link) = ex { - Some(llvm_link) - } else { - None - } - }) - } - - pub fn generate_variants(&self, global_ctx: &GlobalContext) -> context::Result> { - let wrap_err = |err| format!("{}: {err}", self.signature.name); - - let mut group_ctx = GroupContext::default(); - self.input - .variants(self) - .map_err(wrap_err)? - .map(|input| { - self.generate_variant(input.clone(), &mut group_ctx, global_ctx) - .map_err(wrap_err) - .map(|variant| (variant, input)) - }) - .collect::>>() - .and_then(|mut variants| { - variants.sort_by_cached_key(|(_, input)| input.to_owned()); - - if variants.is_empty() { - let standalone_variant = self - .generate_variant(InputSet::default(), &mut group_ctx, global_ctx) - .map_err(wrap_err)?; - - Ok(vec![standalone_variant]) - } else { - Ok(variants - .into_iter() - .map(|(variant, _)| variant) - .collect_vec()) - } - }) - } - - pub fn generate_variant( - &self, - input: InputSet, - group_ctx: &mut GroupContext, - global_ctx: &GlobalContext, - ) -> context::Result { - let mut variant = self.clone(); - - variant.input.types = vec![InputSetEntry::new(vec![input.clone()])]; - - let mut local_ctx = LocalContext::new(input, self); - let mut ctx = Context { - local: &mut local_ctx, - group: group_ctx, - global: global_ctx, - }; - - variant.pre_build(&mut ctx)?; - - match ctx.local.predicate_form().cloned() { - Some(PredicateForm::DontCare(method)) => { - variant.compose = variant.generate_dont_care_pass_through(&mut ctx, method)? - } - Some(PredicateForm::Zeroing(method)) => { - variant.compose = variant.generate_zeroing_pass_through(&mut ctx, method)? - } - _ => { - for idx in 0..variant.compose.len() { - let mut ex = variant.compose[idx].clone(); - ex.build(&variant, &mut ctx)?; - variant.compose[idx] = ex; - } - } - }; - - if variant.attr.is_none() && variant.assert_instr.is_none() { - panic!("Error: {} is missing both 'attr' and 'assert_instr' fields. You must either manually declare the attributes using the 'attr' field or use 'assert_instr'!", variant.signature.name.to_string()); - } - - if variant.attr.is_some() { - let attr: &Vec = &variant.attr.clone().unwrap(); - let mut expanded_attr: Vec = Vec::new(); - for idx in 0..attr.len() { - let mut ex = attr[idx].clone(); - ex.build(&variant, &mut ctx)?; - expanded_attr.push(ex); - } - variant.attr = Some(expanded_attr); - } - - variant.post_build(&mut ctx)?; - - if let Some(n_variant_op) = ctx.local.n_variant_op().cloned() { - variant.generate_n_variant(n_variant_op, &mut ctx) - } else { - Ok(variant) - } - } - - /// Implement a "zeroing" (_z) method by calling an existing "merging" (_m) method, as required. - fn generate_zeroing_pass_through( - &mut self, - ctx: &mut Context, - method: ZeroingMethod, - ) -> context::Result> { - PredicationMask::try_from(&ctx.local.signature.name) - .ok() - .filter(|mask| mask.has_merging()) - .ok_or_else(|| format!("cannot generate zeroing passthrough for {}, no merging predicate form is specified", self.signature.name))?; - - // Determine the function to pass through to. - let mut target_ctx = ctx.local.clone(); - // Change target function predicate form to merging - *target_ctx.input.iter_mut() - .find_map(|arg| arg.predicate_form_mut()) - .expect("failed to generate zeroing pass through, could not find predicate form in the InputSet") = PredicateForm::Merging; - - let mut sig = target_ctx.signature.clone(); - sig.build(&target_ctx)?; - - let args_as_expressions = |arg: &Argument| -> context::Result { - let arg_name = arg.name.to_string(); - match &method { - ZeroingMethod::Drop { drop } if arg_name == drop.to_string() => { - Ok(PredicateForm::make_zeroinitializer(&arg.kind)) - } - ZeroingMethod::Select { select } if arg_name == select.to_string() => { - let pg = sig - .arguments - .iter() - .find_map(|arg| match arg.kind.vector() { - Some(ty) if ty.base_type().is_bool() => Some(arg.name.clone()), - _ => None, - }) - .ok_or_else(|| { - format!("cannot generate zeroing passthrough for {}, no predicate found in the signature for zero selection", self.signature.name) - })?; - Ok(PredicateForm::make_zeroselector( - pg, - select.clone(), - &arg.kind, - )) - } - _ => Ok(arg.into()), - } - }; - - let name: Expression = sig.fn_name().into(); - let args: Vec = sig - .arguments - .iter() - .map(args_as_expressions) - .try_collect()?; - let statics: Vec = sig - .static_defs - .iter() - .map(|sd| sd.try_into()) - .try_collect()?; - let mut call: Expression = FnCall(Box::new(name), args, statics).into(); - call.build(self, ctx)?; - Ok(vec![call]) - } - - /// Implement a "don't care" (_x) method by calling an existing "merging" (_m). - fn generate_dont_care_pass_through( - &mut self, - ctx: &mut Context, - method: DontCareMethod, - ) -> context::Result> { - PredicationMask::try_from(&ctx.local.signature.name).and_then(|mask| match method { - DontCareMethod::AsMerging if mask.has_merging() => Ok(()), - DontCareMethod::AsZeroing if mask.has_zeroing() => Ok(()), - _ => Err(format!( - "cannot generate don't care passthrough for {}, no {} predicate form is specified", - self.signature.name, - match method { - DontCareMethod::AsMerging => "merging", - DontCareMethod::AsZeroing => "zeroing", - _ => unreachable!(), - } - )), - })?; - - // Determine the function to pass through to. - let mut target_ctx = ctx.local.clone(); - // Change target function predicate form to merging - *target_ctx.input.iter_mut() - .find_map(|arg| arg.predicate_form_mut()) - .expect("failed to generate don't care passthrough, could not find predicate form in the InputSet") = PredicateForm::Merging; - - let mut sig = target_ctx.signature.clone(); - sig.build(&target_ctx)?; - - // We might need to drop an argument for a zeroing pass-through. - let drop = match (method, &self.input.predication_methods.zeroing_method) { - (DontCareMethod::AsZeroing, Some(ZeroingMethod::Drop { drop })) => Some(drop), - _ => None, - }; - - let name: Expression = sig.fn_name().into(); - let args: Vec = sig - .arguments - .iter() - .map(|arg| { - if Some(arg.name.to_string()) == drop.as_ref().map(|v| v.to_string()) { - // This argument is present in the _m form, but missing from the _x form. Clang - // typically replaces these with an uninitialised vector, but to avoid - // materialising uninitialised values in Rust, we instead merge with a known - // vector. This usually results in the same code generation. - // TODO: In many cases, it'll be better to use an unpredicated (or zeroing) form. - sig.arguments - .iter() - .filter(|&other| arg.name.to_string() != other.name.to_string()) - .find_map(|other| { - arg.kind.express_reinterpretation_from(&other.kind, other) - }) - .unwrap_or_else(|| PredicateForm::make_zeroinitializer(&arg.kind)) - } else { - arg.into() - } - }) - .collect(); - let statics: Vec = sig - .static_defs - .iter() - .map(|sd| sd.try_into()) - .try_collect()?; - let mut call: Expression = FnCall(Box::new(name), args, statics).into(); - call.build(self, ctx)?; - Ok(vec![call]) - } - - /// Implement a "_n" variant based on the given operand - fn generate_n_variant( - &self, - mut n_variant_op: WildString, - ctx: &mut Context, - ) -> context::Result { - let mut variant = self.clone(); - - n_variant_op.build_acle(ctx.local)?; - - let n_op_arg_idx = variant - .signature - .arguments - .iter_mut() - .position(|arg| arg.name.to_string() == n_variant_op.to_string()) - .ok_or_else(|| { - format!( - "cannot generate `_n` variant for {}, operand `{n_variant_op}` not found", - variant.signature.name - ) - })?; - - let has_n_wildcard = ctx - .local - .signature - .name - .wildcards() - .any(|w| matches!(w, Wildcard::NVariant)); - - if !has_n_wildcard { - return Err(format!("cannot generate `_n` variant for {}, no wildcard {{_n}} was specified in the intrinsic's name", variant.signature.name)); - } - - // Build signature - variant.signature = ctx.local.signature.clone(); - if let Some(pf) = ctx.local.predicate_form() { - // WARN: this may break in the future according to the underlying implementation - // Drops unwanted arguments if needed (required for the collection of arguments to pass to the function) - pf.post_build(&mut variant)?; - } - - let sig = &mut variant.signature; - - ctx.local - .substitutions - .insert(Wildcard::NVariant, "_n".to_owned()); - - let arg_kind = &mut sig.arguments.get_mut(n_op_arg_idx).unwrap().kind; - *arg_kind = match arg_kind { - TypeKind::Wildcard(Wildcard::SVEType(idx, None)) => { - TypeKind::Wildcard(Wildcard::Type(*idx)) - } - _ => { - return Err(format!( - "cannot generate `_n` variant for {}, the given operand is not a valid SVE type", - variant.signature.name - )) - } - }; - - sig.build(ctx.local)?; - - // Build compose - let name: Expression = self.signature.fn_name().into(); - let args: Vec = sig - .arguments - .iter() - .enumerate() - .map(|(idx, arg)| { - let ty = arg.kind.acle_notation_repr(); - if idx == n_op_arg_idx { - FnCall::new_expression( - WildString::from(format!("svdup_n_{ty}")).into(), - vec![arg.into()], - ) - } else { - arg.into() - } - }) - .collect(); - let statics: Vec = sig - .static_defs - .iter() - .map(|sd| sd.try_into()) - .try_collect()?; - let mut call: Expression = FnCall(Box::new(name), args, statics).into(); - call.build(self, ctx)?; - - variant.compose = vec![call]; - variant.signature.predicate_needs_conversion = true; - - Ok(variant) - } - - fn pre_build(&mut self, ctx: &mut Context) -> context::Result { - self.substitutions - .iter_mut() - .try_for_each(|(k, v)| -> context::Result { - let mut ws = v.get(ctx.local)?; - ws.build_acle(ctx.local)?; - ctx.local - .substitutions - .insert(Wildcard::Custom(k.to_owned()), ws.to_string()); - Ok(()) - })?; - - self.signature.build(ctx.local)?; - - if self.safety.is_none() { - self.safety = match Safety::safe_checked(self) { - Ok(safe) => Some(safe), - Err(err) => { - eprintln!("{err}"); - return Err(format!( - "Refusing to infer unsafety for {name}", - name = self.signature.doc_name() - )); - } - } - } - - if let Some(doc) = &mut self.doc { - doc.build_acle(ctx.local)? - } - - // Add arguments to variable tracking - self.signature - .arguments - .iter() - .for_each(|arg| arg.populate_variables(&mut ctx.local.variables)); - - // Add constant expressions to variable tracking - self.signature - .static_defs - .iter() - .filter_map(StaticDefinition::as_variable) - .for_each(|(var_name, var_properties)| { - ctx.local.variables.insert(var_name, var_properties); - }); - - // Pre-build compose expressions - for idx in 0..self.compose.len() { - let mut ex = self.compose[idx].clone(); - ex.pre_build(ctx)?; - self.compose[idx] = ex; - } - - if !ctx.local.input.is_empty() { - // We simplify the LLVM link transmute logic by deferring to a variant employing the same LLVM link where possible - if let Some(link) = self.compose.iter().find_map(|ex| match ex { - Expression::LLVMLink(link) => Some(link), - _ => None, - }) { - let mut link = link.clone(); - link.build(ctx)?; - - for cfg in ctx.global.arch_cfgs.iter() { - let expected_link = link.resolve(cfg); - if let Some(target_inputset) = ctx.group.links.get(&expected_link) { - self.defer_to_existing_llvm_link(ctx.local, target_inputset)?; - break; - } - } - } - } - - if let Some(ref mut assert_instr) = self.assert_instr { - assert_instr.iter_mut().try_for_each(|ai| ai.build(ctx))?; - } - - // Prepend constraint assertions - self.constraints.iter_mut().try_for_each(|c| c.build(ctx))?; - let assertions: Vec<_> = self - .constraints - .iter() - .map(|c| ctx.local.make_assertion_from_constraint(c)) - .try_collect()?; - self.compose.splice(0..0, assertions); - - Ok(()) - } - - fn post_build(&mut self, ctx: &mut Context) -> context::Result { - if let Some(Expression::LLVMLink(link)) = self.compose.last() { - let mut fn_call = link.make_fn_call(&self.signature)?; - // Required to inject conversions - fn_call.build(self, ctx)?; - self.compose.push(fn_call) - } - - if let Some(llvm_link) = self.llvm_link_mut() { - // Turn all Rust unsigned types into signed - llvm_link.sanitise_uints(); - } - - if let Some(predicate_form) = ctx.local.predicate_form() { - predicate_form.post_build(self)? - } - - // Set for ToTokens to display a generic svbool_t - self.signature.predicate_needs_conversion = true; - - // Set base type kind for instruction assertion - self.base_type = ctx - .local - .input - .get(0) - .and_then(|arg| arg.typekind()) - .and_then(|ty| ty.base_type()) - .map(BaseType::clone); - - // Add global target features - self.target_features = ctx - .global - .arch_cfgs - .iter() - .flat_map(|cfg| cfg.target_feature.clone()) - .chain(self.target_features.clone()) - .collect_vec(); - - Ok(()) - } - - fn defer_to_existing_llvm_link( - &mut self, - ctx: &LocalContext, - target_inputset: &InputSet, - ) -> context::Result { - let mut target_ctx = ctx.clone(); - target_ctx.input = target_inputset.clone(); - - let mut target_signature = target_ctx.signature.clone(); - target_signature.build(&target_ctx)?; - - let drop_var = if let Some(pred) = ctx.predicate_form().cloned() { - match pred { - PredicateForm::Zeroing(ZeroingMethod::Drop { drop }) => Some(drop), - PredicateForm::DontCare(DontCareMethod::AsZeroing) => { - if let Some(ZeroingMethod::Drop { drop }) = - self.input.predication_methods.zeroing_method.to_owned() - { - Some(drop) - } else { - None - } - } - _ => None, - } - } else { - None - }; - - let call_method = - |ex, method: &str| Expression::MethodCall(Box::new(ex), method.to_string(), vec![]); - let as_unsigned = |ex| call_method(ex, "as_unsigned"); - let as_signed = |ex| call_method(ex, "as_signed"); - let convert_if_required = |w: Option<&Wildcard>, from: &InputSet, to: &InputSet, ex| { - if let Some(w) = w { - if let Some(dest_idx) = w.get_typeset_index() { - let from_type = from.get(dest_idx); - let to_type = to.get(dest_idx); - - if from_type != to_type { - let from_base_type = from_type - .and_then(|in_arg| in_arg.typekind()) - .and_then(|ty| ty.base_type()) - .map(|bt| bt.kind()); - let to_base_type = to_type - .and_then(|in_arg| in_arg.typekind()) - .and_then(|ty| ty.base_type()) - .map(|bt| bt.kind()); - - match (from_base_type, to_base_type) { - // Use AsSigned for uint -> int - (Some(BaseTypeKind::UInt), Some(BaseTypeKind::Int)) => as_signed(ex), - (Some(BaseTypeKind::Int), Some(BaseTypeKind::Int)) => ex, - // Use AsUnsigned for int -> uint - (Some(BaseTypeKind::Int), Some(BaseTypeKind::UInt)) => as_unsigned(ex), - (Some(BaseTypeKind::Float), Some(BaseTypeKind::Float)) => ex, - (Some(BaseTypeKind::UInt), Some(BaseTypeKind::UInt)) => ex, - (Some(BaseTypeKind::Poly), Some(BaseTypeKind::Poly)) => ex, - - (None, None) => ex, - _ => unreachable!("unsupported conversion case from {from_base_type:?} to {to_base_type:?} hit"), - } - } else { - ex - } - } else { - ex - } - } else { - ex - } - }; - - let args = ctx - .signature - .arguments - .iter() - .filter_map(|arg| { - let var = Expression::Identifier(arg.name.to_owned(), IdentifierType::Variable); - if drop_var.as_ref().map(|v| v.to_string()) != Some(arg.name.to_string()) { - Some(convert_if_required( - arg.kind.wildcard(), - &ctx.input, - target_inputset, - var, - )) - } else { - None - } - }) - .collect_vec(); - - let turbofish = self - .signature - .static_defs - .iter() - .map(|def| { - let name = match def { - StaticDefinition::Constant(Argument { name, .. }) => name.to_string(), - StaticDefinition::Generic(name) => name.to_string(), - }; - Expression::Identifier(name.into(), IdentifierType::Symbol) - }) - .collect_vec(); - - let ret_wildcard = ctx - .signature - .return_type - .as_ref() - .and_then(|t| t.wildcard()); - let call = FnCall(Box::new(target_signature.fn_name().into()), args, turbofish).into(); - - self.compose = vec![convert_if_required( - ret_wildcard, - target_inputset, - &ctx.input, - call, - )]; - - Ok(()) - } -} - -impl ToTokens for Intrinsic { - fn to_tokens(&self, tokens: &mut TokenStream) { - let signature = &self.signature; - let fn_name = signature.fn_name().to_string(); - let target_feature = self.target_features.join(","); - let safety = self - .safety - .as_ref() - .expect("safety should be determined during `pre_build`"); - - if let Some(doc) = &self.doc { - let mut doc = vec![doc.to_string()]; - - doc.push(format!("[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/{})", &signature.doc_name())); - - if safety.has_doc_comments() { - doc.push("## Safety".to_string()); - for comment in safety.doc_comments() { - doc.push(format!(" * {comment}")); - } - } else { - assert!( - safety.is_safe(), - "{fn_name} is both public and unsafe, and so needs safety documentation" - ); - } - - tokens.append_all(quote! { #(#[doc = #doc])* }); - } else { - assert!( - matches!(self.visibility, FunctionVisibility::Private), - "{fn_name} needs to be private, or to have documentation." - ); - assert!( - !safety.has_doc_comments(), - "{fn_name} needs a documentation section for its safety comments." - ); - } - - tokens.append_all(quote! { #[inline] }); - - /* If we have manually defined attributes on the block of yaml with - * 'attr:' we want to add them */ - if let Some(attr) = &self.attr { - /* Scan to see if we have defined `FnCall: [target_feature, ['']]`*/ - if !has_target_feature_attr(attr) { - /* If not add the default one that is defined at the top of - * the yaml file. This does mean we scan the attributes vector - * twice, once to see if the `target_feature` exists and again - * to actually append the tokens. We could impose that the - * `target_feature` call has to be the first argument of the - * `attr` block */ - tokens.append_all(quote! { - #[target_feature(enable = #target_feature)] - }); - } - - /* Target feature will get added here */ - let attr_expressions = &mut attr.iter().peekable(); - while let Some(ex) = attr_expressions.next() { - tokens.append(Punct::new('#', Spacing::Alone)); - tokens.append(Punct::new('[', Spacing::Alone)); - ex.to_tokens(tokens); - tokens.append(Punct::new(']', Spacing::Alone)); - } - } else { - tokens.append_all(quote! { - #[target_feature(enable = #target_feature)] - }); - } - - if let Some(assert_instr) = &self.assert_instr { - if !assert_instr.is_empty() { - InstructionAssertionsForBaseType(&assert_instr, &self.base_type.as_ref()) - .to_tokens(tokens) - } - } - - match &self.visibility { - FunctionVisibility::Public => tokens.append_all(quote! { pub }), - FunctionVisibility::Private => {} - } - if safety.is_unsafe() { - tokens.append_all(quote! { unsafe }); - } - tokens.append_all(quote! { #signature }); - tokens.append(Punct::new('{', Spacing::Alone)); - - let mut body_unsafe = false; - let mut expressions = self.compose.iter().peekable(); - while let Some(ex) = expressions.next() { - if !body_unsafe && safety.is_safe() && ex.requires_unsafe_wrapper(&fn_name) { - body_unsafe = true; - tokens.append_all(quote! { unsafe }); - tokens.append(Punct::new('{', Spacing::Alone)); - } - // If it's not the last and not a LLVM link, add a trailing semicolon - if expressions.peek().is_some() && !matches!(ex, Expression::LLVMLink(_)) { - tokens.append_all(quote! { #ex; }) - } else { - ex.to_tokens(tokens) - } - } - if body_unsafe { - tokens.append(Punct::new('}', Spacing::Alone)); - } - - tokens.append(Punct::new('}', Spacing::Alone)); - tokens.append(Punct::new('\n', Spacing::Alone)); - tokens.append(Punct::new('\n', Spacing::Alone)); - } -} - -fn has_target_feature_attr(attrs: &[Expression]) -> bool { - attrs.iter().any(|attr| { - if let Expression::FnCall(fn_call) = attr { - fn_call.is_target_feature_call() - } else { - false - } - }) -} diff --git a/library/stdarch/crates/stdarch-gen2/src/load_store_tests.rs b/library/stdarch/crates/stdarch-gen2/src/load_store_tests.rs deleted file mode 100644 index d697a8d22d6..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/load_store_tests.rs +++ /dev/null @@ -1,818 +0,0 @@ -use std::fs::File; -use std::io::Write; -use std::path::PathBuf; -use std::str::FromStr; - -use crate::format_code; -use crate::input::InputType; -use crate::intrinsic::Intrinsic; -use crate::typekinds::BaseType; -use crate::typekinds::{ToRepr, TypeKind}; - -use itertools::Itertools; -use lazy_static::lazy_static; -use proc_macro2::TokenStream; -use quote::{format_ident, quote}; - -// Number of vectors in our buffers - the maximum tuple size, 4, plus 1 as we set the vnum -// argument to 1. -const NUM_VECS: usize = 5; -// The maximum vector length (in bits) -const VL_MAX_BITS: usize = 2048; -// The maximum vector length (in bytes) -const VL_MAX_BYTES: usize = VL_MAX_BITS / 8; -// The maximum number of elements in each vector type -const LEN_F32: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_F64: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_I8: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_I16: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_I32: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_I64: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_U8: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_U16: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_U32: usize = VL_MAX_BYTES / core::mem::size_of::(); -const LEN_U64: usize = VL_MAX_BYTES / core::mem::size_of::(); - -/// `load_intrinsics` and `store_intrinsics` is a vector of intrinsics -/// variants, while `out_path` is a file to write to. -pub fn generate_load_store_tests( - load_intrinsics: Vec, - store_intrinsics: Vec, - out_path: Option<&PathBuf>, -) -> Result<(), String> { - let output = match out_path { - Some(out) => { - Box::new(File::create(out).map_err(|e| format!("couldn't create tests file: {e}"))?) - as Box - } - None => Box::new(std::io::stdout()) as Box, - }; - let mut used_stores = vec![false; store_intrinsics.len()]; - let tests: Vec<_> = load_intrinsics - .iter() - .map(|load| { - let store_candidate = load - .signature - .fn_name() - .to_string() - .replace("svld1s", "svst1") - .replace("svld1u", "svst1") - .replace("svldnt1s", "svstnt1") - .replace("svldnt1u", "svstnt1") - .replace("svld", "svst") - .replace("gather", "scatter"); - - let store_index = store_intrinsics - .iter() - .position(|i| i.signature.fn_name().to_string() == store_candidate); - if let Some(i) = store_index { - used_stores[i] = true; - } - - generate_single_test( - load.clone(), - store_index.map(|i| store_intrinsics[i].clone()), - ) - }) - .try_collect()?; - - assert!(used_stores.into_iter().all(|b| b), "Not all store tests have been paired with a load. Consider generating specifc store-only tests"); - - let preamble = - TokenStream::from_str(&PREAMBLE).map_err(|e| format!("Preamble is invalid: {e}"))?; - // Only output manual tests for the SVE set - let manual_tests = match &load_intrinsics[0].target_features[..] { - [s] if s == "sve" => TokenStream::from_str(&MANUAL_TESTS) - .map_err(|e| format!("Manual tests are invalid: {e}"))?, - _ => quote!(), - }; - format_code( - output, - format!( - "// This code is automatically generated. DO NOT MODIFY. -// -// Instead, modify `crates/stdarch-gen2/spec/sve` and run the following command to re-generate this -// file: -// -// ``` -// cargo run --bin=stdarch-gen2 -- crates/stdarch-gen2/spec -// ``` -{}", - quote! { #preamble #(#tests)* #manual_tests } - ), - ) - .map_err(|e| format!("couldn't write tests: {e}")) -} - -/// A test looks like this: -/// ``` -/// let data = [scalable vector]; -/// -/// let mut storage = [0; N]; -/// -/// store_intrinsic([true_predicate], storage.as_mut_ptr(), data); -/// [test contents of storage] -/// -/// let loaded == load_intrinsic([true_predicate], storage.as_ptr()) -/// assert!(loaded == data); -/// ``` -/// We intialise our data such that the value stored matches the index it's stored to. -/// By doing this we can validate scatters by checking that each value in the storage -/// array is either 0 or the same as its index. -fn generate_single_test( - load: Intrinsic, - store: Option, -) -> Result { - let chars = LdIntrCharacteristics::new(&load)?; - let fn_name = load.signature.fn_name().to_string(); - - if let Some(ty) = &chars.gather_bases_type { - if ty.base_type().unwrap().get_size() == Ok(32) - && chars.gather_index_type.is_none() - && chars.gather_offset_type.is_none() - { - // We lack a way to ensure data is in the bottom 32 bits of the address space - println!("Skipping test for {fn_name}"); - return Ok(quote!()); - } - } - - if fn_name.starts_with("svldff1") && fn_name.contains("gather") { - // TODO: We can remove this check when first-faulting gathers are fixed in CI's QEMU - // https://gitlab.com/qemu-project/qemu/-/issues/1612 - println!("Skipping test for {fn_name}"); - return Ok(quote!()); - } - - let fn_ident = format_ident!("{fn_name}"); - let test_name = format_ident!( - "test_{fn_name}{}", - if let Some(ref store) = store { - format!("_with_{}", store.signature.fn_name()) - } else { - String::new() - } - ); - - let load_type = &chars.load_type; - let acle_type = load_type.acle_notation_repr(); - - // If there's no return type, fallback to the load type for things that depend on it - let ret_type = &load - .signature - .return_type - .as_ref() - .and_then(TypeKind::base_type) - .unwrap_or(load_type); - - let pred_fn = format_ident!("svptrue_b{}", load_type.size()); - - let load_type_caps = load_type.rust_repr().to_uppercase(); - let data_array = format_ident!("{load_type_caps}_DATA"); - - let size_fn = format_ident!("svcnt{}", ret_type.size_literal()); - - let rust_ret_type = ret_type.rust_repr(); - let assert_fn = format_ident!("assert_vector_matches_{rust_ret_type}"); - - // Use vnum=1, so adjust all values by one vector length - let (length_call, vnum_arg) = if chars.vnum { - if chars.is_prf { - (quote!(), quote!(, 1)) - } else { - (quote!(let len = #size_fn() as usize;), quote!(, 1)) - } - } else { - (quote!(), quote!()) - }; - - let (bases_load, bases_arg) = if let Some(ty) = &chars.gather_bases_type { - // Bases is a vector of (sometimes 32-bit) pointers - // When we combine bases with an offset/index argument, we load from the data arrays - // starting at 1 - let base_ty = ty.base_type().unwrap(); - let rust_type = format_ident!("{}", base_ty.rust_repr()); - let index_fn = format_ident!("svindex_{}", base_ty.acle_notation_repr()); - let size_in_bytes = chars.load_type.get_size().unwrap() / 8; - - if base_ty.get_size().unwrap() == 32 { - // Treat bases as a vector of offsets here - we don't test this without an offset or - // index argument - ( - Some(quote!( - let bases = #index_fn(0, #size_in_bytes.try_into().unwrap()); - )), - quote!(, bases), - ) - } else { - // Treat bases as a vector of pointers - let base_fn = format_ident!("svdup_n_{}", base_ty.acle_notation_repr()); - let data_array = if store.is_some() { - format_ident!("storage") - } else { - format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) - }; - - let add_fn = format_ident!("svadd_{}_x", base_ty.acle_notation_repr()); - ( - Some(quote! { - let bases = #base_fn(#data_array.as_ptr() as #rust_type); - let offsets = #index_fn(0, #size_in_bytes.try_into().unwrap()); - let bases = #add_fn(#pred_fn(), bases, offsets); - }), - quote!(, bases), - ) - } - } else { - (None, quote!()) - }; - - let index_arg = if let Some(ty) = &chars.gather_index_type { - let rust_type = format_ident!("{}", ty.rust_repr()); - if chars - .gather_bases_type - .as_ref() - .and_then(TypeKind::base_type) - .map_or(Err(String::new()), BaseType::get_size) - .unwrap() - == 32 - { - // Let index be the base of the data array - let data_array = if store.is_some() { - format_ident!("storage") - } else { - format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) - }; - let size_in_bytes = chars.load_type.get_size().unwrap() / 8; - quote!(, #data_array.as_ptr() as #rust_type / (#size_in_bytes as #rust_type) + 1) - } else { - quote!(, 1.try_into().unwrap()) - } - } else { - quote!() - }; - - let offset_arg = if let Some(ty) = &chars.gather_offset_type { - let size_in_bytes = chars.load_type.get_size().unwrap() / 8; - if chars - .gather_bases_type - .as_ref() - .and_then(TypeKind::base_type) - .map_or(Err(String::new()), BaseType::get_size) - .unwrap() - == 32 - { - // Let offset be the base of the data array - let rust_type = format_ident!("{}", ty.rust_repr()); - let data_array = if store.is_some() { - format_ident!("storage") - } else { - format_ident!("{}_DATA", chars.load_type.rust_repr().to_uppercase()) - }; - quote!(, #data_array.as_ptr() as #rust_type + #size_in_bytes as #rust_type) - } else { - quote!(, #size_in_bytes.try_into().unwrap()) - } - } else { - quote!() - }; - - let (offsets_load, offsets_arg) = if let Some(ty) = &chars.gather_offsets_type { - // Offsets is a scalable vector of per-element offsets in bytes. We re-use the contiguous - // data for this, then multiply to get indices - let offsets_fn = format_ident!("svindex_{}", ty.base_type().unwrap().acle_notation_repr()); - let size_in_bytes = chars.load_type.get_size().unwrap() / 8; - ( - Some(quote! { - let offsets = #offsets_fn(0, #size_in_bytes.try_into().unwrap()); - }), - quote!(, offsets), - ) - } else { - (None, quote!()) - }; - - let (indices_load, indices_arg) = if let Some(ty) = &chars.gather_indices_type { - // There's no need to multiply indices by the load type width - let base_ty = ty.base_type().unwrap(); - let indices_fn = format_ident!("svindex_{}", base_ty.acle_notation_repr()); - ( - Some(quote! { - let indices = #indices_fn(0, 1); - }), - quote! {, indices}, - ) - } else { - (None, quote!()) - }; - - let ptr = if chars.gather_bases_type.is_some() { - quote!() - } else if chars.is_prf { - quote!(, I64_DATA.as_ptr()) - } else { - quote!(, #data_array.as_ptr()) - }; - - let tuple_len = &chars.tuple_len; - let expecteds = if chars.is_prf { - // No return value for prefetches - vec![] - } else { - (0..*tuple_len) - .map(|i| get_expected_range(i, &chars)) - .collect() - }; - let asserts: Vec<_> = - if *tuple_len > 1 { - let svget = format_ident!("svget{tuple_len}_{acle_type}"); - expecteds.iter().enumerate().map(|(i, expected)| { - quote! (#assert_fn(#svget::<{ #i as i32 }>(loaded), #expected);) - }).collect() - } else { - expecteds - .iter() - .map(|expected| quote! (#assert_fn(loaded, #expected);)) - .collect() - }; - - let function = if chars.is_prf { - if fn_name.contains("gather") && fn_name.contains("base") && !fn_name.starts_with("svprf_") - { - // svprf(b|h|w|d)_gather base intrinsics do not have a generic type parameter - quote!(#fn_ident::<{ svprfop::SV_PLDL1KEEP }>) - } else { - quote!(#fn_ident::<{ svprfop::SV_PLDL1KEEP }, i64>) - } - } else { - quote!(#fn_ident) - }; - - let octaword_guard = if chars.replicate_width == Some(256) { - let msg = format!("Skipping {test_name} due to SVE vector length"); - quote! { - if svcntb() < 32 { - println!(#msg); - return; - } - } - } else { - quote!() - }; - - let feats = load.target_features.join(","); - - if let Some(store) = store { - let data_init = if *tuple_len == 1 { - quote!(#(#expecteds)*) - } else { - let create = format_ident!("svcreate{tuple_len}_{acle_type}"); - quote!(#create(#(#expecteds),*)) - }; - let input = store.input.types.get(0).unwrap().get(0).unwrap(); - let store_type = input - .get(store.test.get_typeset_index().unwrap()) - .and_then(InputType::typekind) - .and_then(TypeKind::base_type) - .unwrap(); - - let store_type = format_ident!("{}", store_type.rust_repr()); - let storage_len = NUM_VECS * VL_MAX_BITS / chars.load_type.get_size()? as usize; - let store_fn = format_ident!("{}", store.signature.fn_name().to_string()); - let load_type = format_ident!("{}", chars.load_type.rust_repr()); - let (store_ptr, store_mut_ptr) = if chars.gather_bases_type.is_none() { - ( - quote!(, storage.as_ptr() as *const #load_type), - quote!(, storage.as_mut_ptr()), - ) - } else { - (quote!(), quote!()) - }; - let args = quote!(#pred_fn() #store_ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg); - let call = if chars.uses_ffr { - // Doing a normal load first maximises the number of elements our ff/nf test loads - let non_ffr_fn_name = format_ident!( - "{}", - fn_name - .replace("svldff1", "svld1") - .replace("svldnf1", "svld1") - ); - quote! { - svsetffr(); - let _ = #non_ffr_fn_name(#args); - let loaded = #function(#args); - } - } else { - // Note that the FFR must be set for all tests as the assert functions mask against it - quote! { - svsetffr(); - let loaded = #function(#args); - } - }; - - Ok(quote! { - #[simd_test(enable = #feats)] - unsafe fn #test_name() { - #octaword_guard - #length_call - let mut storage = [0 as #store_type; #storage_len]; - let data = #data_init; - #bases_load - #offsets_load - #indices_load - - #store_fn(#pred_fn() #store_mut_ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg, data); - for (i, &val) in storage.iter().enumerate() { - assert!(val == 0 as #store_type || val == i as #store_type); - } - - #call - #(#asserts)* - - } - }) - } else { - let args = quote!(#pred_fn() #ptr #vnum_arg #bases_arg #offset_arg #index_arg #offsets_arg #indices_arg); - let call = if chars.uses_ffr { - // Doing a normal load first maximises the number of elements our ff/nf test loads - let non_ffr_fn_name = format_ident!( - "{}", - fn_name - .replace("svldff1", "svld1") - .replace("svldnf1", "svld1") - ); - quote! { - svsetffr(); - let _ = #non_ffr_fn_name(#args); - let loaded = #function(#args); - } - } else { - // Note that the FFR must be set for all tests as the assert functions mask against it - quote! { - svsetffr(); - let loaded = #function(#args); - } - }; - Ok(quote! { - #[simd_test(enable = #feats)] - unsafe fn #test_name() { - #octaword_guard - #bases_load - #offsets_load - #indices_load - #call - #length_call - - #(#asserts)* - } - }) - } -} - -/// Assumes chars.ret_type is not None -fn get_expected_range(tuple_idx: usize, chars: &LdIntrCharacteristics) -> proc_macro2::TokenStream { - // vnum=1 - let vnum_adjust = if chars.vnum { quote!(len+) } else { quote!() }; - - let bases_adjust = - (chars.gather_index_type.is_some() || chars.gather_offset_type.is_some()) as usize; - - let tuple_len = chars.tuple_len; - let size = chars - .ret_type - .as_ref() - .and_then(TypeKind::base_type) - .unwrap_or(&chars.load_type) - .get_size() - .unwrap() as usize; - - if chars.replicate_width == Some(128) { - // svld1rq - let ty_rust = format_ident!( - "{}", - chars - .ret_type - .as_ref() - .unwrap() - .base_type() - .unwrap() - .rust_repr() - ); - let args: Vec<_> = (0..(128 / size)).map(|i| quote!(#i as #ty_rust)).collect(); - let dup = format_ident!( - "svdupq_n_{}", - chars.ret_type.as_ref().unwrap().acle_notation_repr() - ); - quote!(#dup(#(#args,)*)) - } else if chars.replicate_width == Some(256) { - // svld1ro - we use two interleaved svdups to create a repeating 256-bit pattern - let ty_rust = format_ident!( - "{}", - chars - .ret_type - .as_ref() - .unwrap() - .base_type() - .unwrap() - .rust_repr() - ); - let ret_acle = chars.ret_type.as_ref().unwrap().acle_notation_repr(); - let args: Vec<_> = (0..(128 / size)).map(|i| quote!(#i as #ty_rust)).collect(); - let args2: Vec<_> = ((128 / size)..(256 / size)) - .map(|i| quote!(#i as #ty_rust)) - .collect(); - let dup = format_ident!("svdupq_n_{ret_acle}"); - let interleave = format_ident!("svtrn1q_{ret_acle}"); - quote!(#interleave(#dup(#(#args,)*), #dup(#(#args2,)*))) - } else { - let start = bases_adjust + tuple_idx; - if chars - .ret_type - .as_ref() - .unwrap() - .base_type() - .unwrap() - .is_float() - { - // Use svcvt to create a linear sequence of floats - let cvt_fn = format_ident!("svcvt_f{size}_s{size}_x"); - let pred_fn = format_ident!("svptrue_b{size}"); - let svindex_fn = format_ident!("svindex_s{size}"); - quote! { #cvt_fn(#pred_fn(), #svindex_fn((#vnum_adjust #start).try_into().unwrap(), #tuple_len.try_into().unwrap()))} - } else { - let ret_acle = chars.ret_type.as_ref().unwrap().acle_notation_repr(); - let svindex = format_ident!("svindex_{ret_acle}"); - quote!(#svindex((#vnum_adjust #start).try_into().unwrap(), #tuple_len.try_into().unwrap())) - } - } -} - -struct LdIntrCharacteristics { - // The data type to load from (not necessarily the data type returned) - load_type: BaseType, - // The data type to return (None for unit) - ret_type: Option, - // The size of tuple to load/store - tuple_len: usize, - // Whether a vnum argument is present - vnum: bool, - // Is the intrinsic first/non-faulting? - uses_ffr: bool, - // Is it a prefetch? - is_prf: bool, - // The size of data loaded with svld1ro/q intrinsics - replicate_width: Option, - // Scalable vector of pointers to load from - gather_bases_type: Option, - // Scalar offset, paired with bases - gather_offset_type: Option, - // Scalar index, paired with bases - gather_index_type: Option, - // Scalable vector of offsets - gather_offsets_type: Option, - // Scalable vector of indices - gather_indices_type: Option, -} - -impl LdIntrCharacteristics { - fn new(intr: &Intrinsic) -> Result { - let input = intr.input.types.get(0).unwrap().get(0).unwrap(); - let load_type = input - .get(intr.test.get_typeset_index().unwrap()) - .and_then(InputType::typekind) - .and_then(TypeKind::base_type) - .unwrap(); - - let ret_type = intr.signature.return_type.clone(); - - let name = intr.signature.fn_name().to_string(); - let tuple_len = name - .chars() - .find(|c| c.is_numeric()) - .and_then(|c| c.to_digit(10)) - .unwrap_or(1) as usize; - - let uses_ffr = name.starts_with("svldff") || name.starts_with("svldnf"); - - let is_prf = name.starts_with("svprf"); - - let replicate_width = if name.starts_with("svld1ro") { - Some(256) - } else if name.starts_with("svld1rq") { - Some(128) - } else { - None - }; - - let get_ty_of_arg = |name: &str| { - intr.signature - .arguments - .iter() - .find(|a| a.name.to_string() == name) - .map(|a| a.kind.clone()) - }; - - let gather_bases_type = get_ty_of_arg("bases"); - let gather_offset_type = get_ty_of_arg("offset"); - let gather_index_type = get_ty_of_arg("index"); - let gather_offsets_type = get_ty_of_arg("offsets"); - let gather_indices_type = get_ty_of_arg("indices"); - - Ok(LdIntrCharacteristics { - load_type: *load_type, - ret_type, - tuple_len, - vnum: name.contains("vnum"), - uses_ffr, - is_prf, - replicate_width, - gather_bases_type, - gather_offset_type, - gather_index_type, - gather_offsets_type, - gather_indices_type, - }) - } -} - -lazy_static! { - static ref PREAMBLE: String = format!( - r#"#![allow(unused)] - -use super::*; -use std::boxed::Box; -use std::convert::{{TryFrom, TryInto}}; -use std::sync::LazyLock; -use std::vec::Vec; -use stdarch_test::simd_test; - -static F32_DATA: LazyLock<[f32; {LEN_F32} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_F32} * {NUM_VECS}) - .map(|i| i as f32) - .collect::>() - .try_into() - .expect("f32 data incorrectly initialised") -}}); -static F64_DATA: LazyLock<[f64; {LEN_F64} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_F64} * {NUM_VECS}) - .map(|i| i as f64) - .collect::>() - .try_into() - .expect("f64 data incorrectly initialised") -}}); -static I8_DATA: LazyLock<[i8; {LEN_I8} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_I8} * {NUM_VECS}) - .map(|i| ((i + 128) % 256 - 128) as i8) - .collect::>() - .try_into() - .expect("i8 data incorrectly initialised") -}}); -static I16_DATA: LazyLock<[i16; {LEN_I16} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_I16} * {NUM_VECS}) - .map(|i| i as i16) - .collect::>() - .try_into() - .expect("i16 data incorrectly initialised") -}}); -static I32_DATA: LazyLock<[i32; {LEN_I32} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_I32} * {NUM_VECS}) - .map(|i| i as i32) - .collect::>() - .try_into() - .expect("i32 data incorrectly initialised") -}}); -static I64_DATA: LazyLock<[i64; {LEN_I64} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_I64} * {NUM_VECS}) - .map(|i| i as i64) - .collect::>() - .try_into() - .expect("i64 data incorrectly initialised") -}}); -static U8_DATA: LazyLock<[u8; {LEN_U8} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_U8} * {NUM_VECS}) - .map(|i| i as u8) - .collect::>() - .try_into() - .expect("u8 data incorrectly initialised") -}}); -static U16_DATA: LazyLock<[u16; {LEN_U16} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_U16} * {NUM_VECS}) - .map(|i| i as u16) - .collect::>() - .try_into() - .expect("u16 data incorrectly initialised") -}}); -static U32_DATA: LazyLock<[u32; {LEN_U32} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_U32} * {NUM_VECS}) - .map(|i| i as u32) - .collect::>() - .try_into() - .expect("u32 data incorrectly initialised") -}}); -static U64_DATA: LazyLock<[u64; {LEN_U64} * {NUM_VECS}]> = LazyLock::new(|| {{ - (0..{LEN_U64} * {NUM_VECS}) - .map(|i| i as u64) - .collect::>() - .try_into() - .expect("u64 data incorrectly initialised") -}}); - -#[target_feature(enable = "sve")] -fn assert_vector_matches_f32(vector: svfloat32_t, expected: svfloat32_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b32(), defined)); - let cmp = svcmpne_f32(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_f64(vector: svfloat64_t, expected: svfloat64_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b64(), defined)); - let cmp = svcmpne_f64(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_i8(vector: svint8_t, expected: svint8_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b8(), defined)); - let cmp = svcmpne_s8(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_i16(vector: svint16_t, expected: svint16_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b16(), defined)); - let cmp = svcmpne_s16(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_i32(vector: svint32_t, expected: svint32_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b32(), defined)); - let cmp = svcmpne_s32(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_i64(vector: svint64_t, expected: svint64_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b64(), defined)); - let cmp = svcmpne_s64(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_u8(vector: svuint8_t, expected: svuint8_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b8(), defined)); - let cmp = svcmpne_u8(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_u16(vector: svuint16_t, expected: svuint16_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b16(), defined)); - let cmp = svcmpne_u16(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_u32(vector: svuint32_t, expected: svuint32_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b32(), defined)); - let cmp = svcmpne_u32(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} - -#[target_feature(enable = "sve")] -fn assert_vector_matches_u64(vector: svuint64_t, expected: svuint64_t) {{ - let defined = svrdffr(); - assert!(svptest_first(svptrue_b64(), defined)); - let cmp = svcmpne_u64(defined, vector, expected); - assert!(!svptest_any(defined, cmp)) -}} -"# - ); -} - -lazy_static! { - static ref MANUAL_TESTS: String = format!( - "#[simd_test(enable = \"sve\")] -unsafe fn test_ffr() {{ - svsetffr(); - let ffr = svrdffr(); - assert_vector_matches_u8(svdup_n_u8_z(ffr, 1), svindex_u8(1, 0)); - let pred = svdupq_n_b8(true, false, true, false, true, false, true, false, - true, false, true, false, true, false, true, false); - svwrffr(pred); - let ffr = svrdffr_z(svptrue_b8()); - assert_vector_matches_u8(svdup_n_u8_z(ffr, 1), svdup_n_u8_z(pred, 1)); -}} -" - ); -} diff --git a/library/stdarch/crates/stdarch-gen2/src/main.rs b/library/stdarch/crates/stdarch-gen2/src/main.rs deleted file mode 100644 index 22bf6724b02..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/main.rs +++ /dev/null @@ -1,279 +0,0 @@ -#![feature(pattern)] - -mod assert_instr; -mod context; -mod expression; -mod fn_suffix; -mod input; -mod intrinsic; -mod load_store_tests; -mod matching; -mod predicate_forms; -mod typekinds; -mod wildcards; -mod wildstring; - -use intrinsic::Test; -use itertools::Itertools; -use quote::quote; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use walkdir::WalkDir; - -fn main() -> Result<(), String> { - parse_args() - .into_iter() - .map(|(filepath, out)| { - File::open(&filepath) - .map(|f| (f, filepath, out)) - .map_err(|e| format!("could not read input file: {e}")) - }) - .map(|res| { - let (file, filepath, out) = res?; - serde_yaml::from_reader(file) - .map(|input: input::GeneratorInput| (input, filepath, out)) - .map_err(|e| format!("could not parse input file: {e}")) - }) - .collect::, _>>()? - .into_iter() - .map(|(input, filepath, out)| { - let intrinsics = input.intrinsics.into_iter() - .map(|intrinsic| { - intrinsic.generate_variants(&input.ctx) - }) - .try_collect() - .map(|mut vv: Vec<_>| { - vv.sort_by_cached_key(|variants| { - variants.first().map_or_else(String::default, |variant| { - variant.signature.fn_name().to_string() - }) - }); - vv.into_iter().flatten().collect_vec() - })?; - - if filepath.ends_with("sve.spec.yml") || filepath.ends_with("sve2.spec.yml") { - let loads = intrinsics.iter() - .filter_map(|i| { - if matches!(i.test, Test::Load(..)) { - Some(i.clone()) - } else { - None - } - }).collect(); - let stores = intrinsics.iter() - .filter_map(|i| { - if matches!(i.test, Test::Store(..)) { - Some(i.clone()) - } else { - None - } - }).collect(); - load_store_tests::generate_load_store_tests(loads, stores, out.as_ref().map(|o| make_tests_filepath(&filepath, o)).as_ref())?; - } - - Ok(( - input::GeneratorInput { - intrinsics, - ctx: input.ctx, - }, - filepath, - out, - )) - }) - .try_for_each( - |result: context::Result<(input::GeneratorInput, PathBuf, Option)>| -> context::Result { - let (generated, filepath, out) = result?; - - let w = match out { - Some(out) => Box::new( - File::create(make_output_filepath(&filepath, &out)) - .map_err(|e| format!("could not create output file: {e}"))?, - ) as Box, - None => Box::new(std::io::stdout()) as Box, - }; - - generate_file(generated, w) - .map_err(|e| format!("could not generate output file: {e}")) - }, - ) -} - -fn parse_args() -> Vec<(PathBuf, Option)> { - let mut args_it = std::env::args().skip(1); - assert!( - 1 <= args_it.len() && args_it.len() <= 2, - "Usage: cargo run -p stdarch-gen2 -- INPUT_DIR [OUTPUT_DIR]" - ); - - let in_path = Path::new(args_it.next().unwrap().as_str()).to_path_buf(); - assert!( - in_path.exists() && in_path.is_dir(), - "invalid path {in_path:#?} given" - ); - - let out_dir = if let Some(dir) = args_it.next() { - let out_path = Path::new(dir.as_str()).to_path_buf(); - assert!( - out_path.exists() && out_path.is_dir(), - "invalid path {out_path:#?} given" - ); - Some(out_path) - } else { - std::env::current_exe() - .map(|mut f| { - f.pop(); - f.push("../../crates/core_arch/src/aarch64/"); - f.exists().then_some(f) - }) - .ok() - .flatten() - }; - - WalkDir::new(in_path) - .into_iter() - .filter_map(Result::ok) - .filter(|f| f.file_type().is_file()) - .map(|f| (f.into_path(), out_dir.clone())) - .collect() -} - -fn generate_file( - generated_input: input::GeneratorInput, - mut out: Box, -) -> std::io::Result<()> { - write!( - out, - r#"// This code is automatically generated. DO NOT MODIFY. -// -// Instead, modify `crates/stdarch-gen2/spec/` and run the following command to re-generate this file: -// -// ``` -// cargo run --bin=stdarch-gen2 -- crates/stdarch-gen2/spec -// ``` -#![allow(improper_ctypes)] - -#[cfg(test)] -use stdarch_test::assert_instr; - -use super::*;{uses_neon} - -"#, - uses_neon = generated_input - .ctx - .uses_neon_types - .then_some("\nuse crate::core_arch::arch::aarch64::*;") - .unwrap_or_default(), - )?; - let intrinsics = generated_input.intrinsics; - format_code(out, quote! { #(#intrinsics)* })?; - Ok(()) -} - -pub fn format_code( - mut output: impl std::io::Write, - input: impl std::fmt::Display, -) -> std::io::Result<()> { - let proc = Command::new("rustfmt") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn()?; - write!(proc.stdin.as_ref().unwrap(), "{input}")?; - output.write_all(proc.wait_with_output()?.stdout.as_slice()) -} - -/// Derive an output file name from an input file and an output directory. -/// -/// The name is formed by: -/// -/// - ... taking in_filepath.file_name() (dropping all directory components), -/// - ... dropping a .yml or .yaml extension (if present), -/// - ... then dropping a .spec extension (if present). -/// -/// Panics if the resulting name is empty, or if file_name() is not UTF-8. -fn make_output_filepath(in_filepath: &Path, out_dirpath: &Path) -> PathBuf { - make_filepath(in_filepath, out_dirpath, |name: &str| format!("{name}.rs")) -} - -fn make_tests_filepath(in_filepath: &Path, out_dirpath: &Path) -> PathBuf { - make_filepath(in_filepath, out_dirpath, |name: &str| { - format!("ld_st_tests_{name}.rs") - }) -} - -fn make_filepath String>( - in_filepath: &Path, - out_dirpath: &Path, - name_formatter: F, -) -> PathBuf { - let mut parts = in_filepath.iter(); - let name = parts - .next_back() - .and_then(|f| f.to_str()) - .expect("Inputs must have valid, UTF-8 file_name()"); - let dir = parts.next_back().unwrap(); - - let name = name - .trim_end_matches(".yml") - .trim_end_matches(".yaml") - .trim_end_matches(".spec"); - assert!(!name.is_empty()); - - let mut output = out_dirpath.to_path_buf(); - output.push(dir); - output.push(name_formatter(name)); - output -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn infer_output_file() { - macro_rules! t { - ($src:expr, $outdir:expr, $dst:expr) => { - let src: PathBuf = $src.iter().collect(); - let outdir: PathBuf = $outdir.iter().collect(); - let dst: PathBuf = $dst.iter().collect(); - assert_eq!(make_output_filepath(&src, &outdir), dst); - }; - } - // Documented usage. - t!(["x", "NAME.spec.yml"], [""], ["x", "NAME.rs"]); - t!( - ["x", "NAME.spec.yml"], - ["a", "b"], - ["a", "b", "x", "NAME.rs"] - ); - t!( - ["x", "y", "NAME.spec.yml"], - ["out"], - ["out", "y", "NAME.rs"] - ); - t!(["x", "NAME.spec.yaml"], ["out"], ["out", "x", "NAME.rs"]); - t!(["x", "NAME.spec"], ["out"], ["out", "x", "NAME.rs"]); - t!(["x", "NAME.yml"], ["out"], ["out", "x", "NAME.rs"]); - t!(["x", "NAME.yaml"], ["out"], ["out", "x", "NAME.rs"]); - // Unrecognised extensions get treated as part of the stem. - t!( - ["x", "NAME.spac.yml"], - ["out"], - ["out", "x", "NAME.spac.rs"] - ); - t!(["x", "NAME.txt"], ["out"], ["out", "x", "NAME.txt.rs"]); - // Always take the top-level directory from the input path - t!( - ["x", "y", "z", "NAME.spec.yml"], - ["out"], - ["out", "z", "NAME.rs"] - ); - } - - #[test] - #[should_panic] - fn infer_output_file_no_stem() { - make_output_filepath(Path::new(".spec.yml"), Path::new("")); - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/matching.rs b/library/stdarch/crates/stdarch-gen2/src/matching.rs deleted file mode 100644 index 0c480620428..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/matching.rs +++ /dev/null @@ -1,170 +0,0 @@ -use proc_macro2::TokenStream; -use quote::ToTokens; -use serde::{Deserialize, Serialize}; -use std::fmt; - -use crate::context::{self, LocalContext}; -use crate::typekinds::{BaseType, BaseTypeKind, TypeKind}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MatchSizeValues { - pub default: T, - pub byte: Option, - pub halfword: Option, - pub doubleword: Option, -} - -impl MatchSizeValues { - pub fn get(&mut self, ty: &TypeKind, ctx: &LocalContext) -> context::Result<&T> { - let base_ty = if let Some(w) = ty.wildcard() { - ctx.provide_type_wildcard(w)? - } else { - ty.clone() - }; - - if let BaseType::Sized(_, bitsize) = base_ty.base_type().unwrap() { - match (bitsize, &self.byte, &self.halfword, &self.doubleword) { - (64, _, _, Some(v)) | (16, _, Some(v), _) | (8, Some(v), _, _) => Ok(v), - _ => Ok(&self.default), - } - } else { - Err(format!("cannot match bitsize to unsized type {ty:?}!")) - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MatchKindValues { - pub default: T, - pub float: Option, - pub unsigned: Option, -} - -impl MatchKindValues { - pub fn get(&mut self, ty: &TypeKind, ctx: &LocalContext) -> context::Result<&T> { - let base_ty = if let Some(w) = ty.wildcard() { - ctx.provide_type_wildcard(w)? - } else { - ty.clone() - }; - - match ( - base_ty.base_type().unwrap().kind(), - &self.float, - &self.unsigned, - ) { - (BaseTypeKind::Float, Some(v), _) | (BaseTypeKind::UInt, _, Some(v)) => Ok(v), - _ => Ok(&self.default), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged, deny_unknown_fields)] -pub enum SizeMatchable { - Matched(T), - Unmatched { - match_size: Option, - #[serde(flatten)] - values: MatchSizeValues>, - }, -} - -impl SizeMatchable { - pub fn perform_match(&mut self, ctx: &LocalContext) -> context::Result { - match self { - Self::Unmatched { - match_size: None, - values: MatchSizeValues { default, .. }, - } => *self = Self::Matched(*default.to_owned()), - Self::Unmatched { - match_size: Some(ty), - values, - } => *self = Self::Matched(*values.get(ty, ctx)?.to_owned()), - _ => {} - } - Ok(()) - } -} - -impl AsRef for SizeMatchable { - fn as_ref(&self) -> &T { - if let SizeMatchable::Matched(v) = self { - v - } else { - panic!("no match for {self:?} was performed"); - } - } -} - -impl AsMut for SizeMatchable { - fn as_mut(&mut self) -> &mut T { - if let SizeMatchable::Matched(v) = self { - v - } else { - panic!("no match for {self:?} was performed"); - } - } -} - -impl ToTokens for SizeMatchable { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.as_ref().to_tokens(tokens) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged, deny_unknown_fields)] -pub enum KindMatchable { - Matched(T), - Unmatched { - match_kind: Option, - #[serde(flatten)] - values: MatchKindValues>, - }, -} - -impl KindMatchable { - pub fn perform_match(&mut self, ctx: &LocalContext) -> context::Result { - match self { - Self::Unmatched { - match_kind: None, - values: MatchKindValues { default, .. }, - } => *self = Self::Matched(*default.to_owned()), - Self::Unmatched { - match_kind: Some(ty), - values, - } => *self = Self::Matched(*values.get(ty, ctx)?.to_owned()), - _ => {} - } - Ok(()) - } -} - -impl AsRef for KindMatchable { - fn as_ref(&self) -> &T { - if let KindMatchable::Matched(v) = self { - v - } else { - panic!("no match for {self:?} was performed"); - } - } -} - -impl AsMut for KindMatchable { - fn as_mut(&mut self) -> &mut T { - if let KindMatchable::Matched(v) = self { - v - } else { - panic!("no match for {self:?} was performed"); - } - } -} - -impl ToTokens for KindMatchable { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.as_ref().to_tokens(tokens) - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/predicate_forms.rs b/library/stdarch/crates/stdarch-gen2/src/predicate_forms.rs deleted file mode 100644 index 02789bf7eb0..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/predicate_forms.rs +++ /dev/null @@ -1,249 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::fmt; -use std::str::FromStr; - -use crate::context; -use crate::expression::{Expression, FnCall, IdentifierType}; -use crate::intrinsic::Intrinsic; -use crate::typekinds::{ToRepr, TypeKind}; -use crate::wildcards::Wildcard; -use crate::wildstring::WildString; - -const ZEROING_SUFFIX: &str = "_z"; -const MERGING_SUFFIX: &str = "_m"; -const DONT_CARE_SUFFIX: &str = "_x"; - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ZeroingMethod { - /// Drop the specified argument and replace it with a zeroinitializer - Drop { drop: WildString }, - /// Apply zero selection to the specified variable when zeroing - Select { select: WildString }, -} - -impl PartialOrd for ZeroingMethod { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for ZeroingMethod { - fn cmp(&self, _: &Self) -> std::cmp::Ordering { - std::cmp::Ordering::Equal - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum DontCareMethod { - #[default] - Inferred, - AsZeroing, - AsMerging, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, Serialize)] -pub struct PredicationMethods { - /// Zeroing method, if the zeroing predicate form is used - #[serde(default)] - pub zeroing_method: Option, - /// Don't care method, if the don't care predicate form is used - #[serde(default)] - pub dont_care_method: DontCareMethod, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum PredicateForm { - /// Enables merging predicate form - Merging, - /// Enables "don't care" predicate form. - DontCare(DontCareMethod), - /// Enables zeroing predicate form. If LLVM zeroselection is performed, then - /// set the `select` field to the variable that gets set. Otherwise set the - /// `drop` field if the zeroinitializer replaces a predicate when merging. - Zeroing(ZeroingMethod), -} - -impl PredicateForm { - pub fn get_suffix(&self) -> &'static str { - match self { - PredicateForm::Zeroing { .. } => ZEROING_SUFFIX, - PredicateForm::Merging => MERGING_SUFFIX, - PredicateForm::DontCare { .. } => DONT_CARE_SUFFIX, - } - } - - pub fn make_zeroinitializer(ty: &TypeKind) -> Expression { - FnCall::new_expression( - format!("svdup_n_{}", ty.acle_notation_repr()) - .parse() - .unwrap(), - vec![if ty.base_type().unwrap().is_float() { - Expression::FloatConstant(0.0) - } else { - Expression::IntConstant(0) - }], - ) - } - - pub fn make_zeroselector(pg_var: WildString, op_var: WildString, ty: &TypeKind) -> Expression { - FnCall::new_expression( - format!("svsel_{}", ty.acle_notation_repr()) - .parse() - .unwrap(), - vec![ - Expression::Identifier(pg_var, IdentifierType::Variable), - Expression::Identifier(op_var, IdentifierType::Variable), - Self::make_zeroinitializer(ty), - ], - ) - } - - pub fn post_build(&self, intrinsic: &mut Intrinsic) -> context::Result { - // Drop the argument - match self { - PredicateForm::Zeroing(ZeroingMethod::Drop { drop: drop_var }) => { - intrinsic.signature.drop_argument(drop_var)? - } - PredicateForm::DontCare(DontCareMethod::AsZeroing) => { - if let ZeroingMethod::Drop { drop } = intrinsic - .input - .predication_methods - .zeroing_method - .to_owned() - .ok_or_else(|| { - "DontCareMethod::AsZeroing without zeroing method.".to_string() - })? - { - intrinsic.signature.drop_argument(&drop)? - } - } - _ => {} - } - - Ok(()) - } - - fn infer_dont_care(mask: &PredicationMask, methods: &PredicationMethods) -> PredicateForm { - let method = if methods.dont_care_method == DontCareMethod::Inferred { - if mask.has_zeroing() - && matches!(methods.zeroing_method, Some(ZeroingMethod::Drop { .. })) - { - DontCareMethod::AsZeroing - } else { - DontCareMethod::AsMerging - } - } else { - methods.dont_care_method - }; - - PredicateForm::DontCare(method) - } - - pub fn compile_list( - mask: &PredicationMask, - methods: &PredicationMethods, - ) -> context::Result> { - let mut forms = Vec::new(); - - if mask.has_merging() { - forms.push(PredicateForm::Merging) - } - - if mask.has_dont_care() { - forms.push(Self::infer_dont_care(mask, methods)) - } - - if mask.has_zeroing() { - if let Some(method) = methods.zeroing_method.to_owned() { - forms.push(PredicateForm::Zeroing(method)) - } else { - return Err( - "cannot create a zeroing variant without a zeroing method specified!" - .to_string(), - ); - } - } - - Ok(forms) - } -} - -#[derive( - Debug, Clone, Copy, Default, PartialEq, Eq, Hash, DeserializeFromStr, SerializeDisplay, -)] -pub struct PredicationMask { - /// Merging - m: bool, - /// Don't care - x: bool, - /// Zeroing - z: bool, -} - -impl PredicationMask { - pub fn has_merging(&self) -> bool { - self.m - } - - pub fn has_dont_care(&self) -> bool { - self.x - } - - pub fn has_zeroing(&self) -> bool { - self.z - } -} - -impl FromStr for PredicationMask { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut result = Self::default(); - for kind in s.bytes() { - match kind { - b'm' => result.m = true, - b'x' => result.x = true, - b'z' => result.z = true, - _ => { - return Err(format!( - "unknown predicate form modifier: {}", - char::from(kind) - )); - } - } - } - - if result.m || result.x || result.z { - Ok(result) - } else { - Err("invalid predication mask".to_string()) - } - } -} - -impl fmt::Display for PredicationMask { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.m.then(|| write!(f, "m")).transpose()?; - self.x.then(|| write!(f, "x")).transpose()?; - self.z.then(|| write!(f, "z")).transpose().map(|_| ()) - } -} - -impl TryFrom<&WildString> for PredicationMask { - type Error = String; - - fn try_from(value: &WildString) -> Result { - value - .wildcards() - .find_map(|w| { - if let Wildcard::PredicateForms(mask) = w { - Some(*mask) - } else { - None - } - }) - .ok_or_else(|| "no predicate forms were specified in the name".to_string()) - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/typekinds.rs b/library/stdarch/crates/stdarch-gen2/src/typekinds.rs deleted file mode 100644 index 7a4fed85ce7..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/typekinds.rs +++ /dev/null @@ -1,1051 +0,0 @@ -use lazy_static::lazy_static; -use proc_macro2::TokenStream; -use quote::{quote, ToTokens, TokenStreamExt}; -use regex::Regex; -use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::fmt; -use std::str::FromStr; - -use crate::context; -use crate::expression::{Expression, FnCall}; -use crate::intrinsic::AccessLevel; -use crate::wildcards::Wildcard; - -const VECTOR_FULL_REGISTER_SIZE: u32 = 128; -const VECTOR_HALF_REGISTER_SIZE: u32 = VECTOR_FULL_REGISTER_SIZE / 2; - -#[derive(Debug, Clone, Copy)] -pub enum TypeRepr { - C, - Rust, - LLVMMachine, - ACLENotation, - Size, - SizeLiteral, - TypeKind, - SizeInBytesLog2, -} - -pub trait ToRepr { - fn repr(&self, repr: TypeRepr) -> String; - - fn c_repr(&self) -> String { - self.repr(TypeRepr::C) - } - - fn rust_repr(&self) -> String { - self.repr(TypeRepr::Rust) - } - - fn llvm_machine_repr(&self) -> String { - self.repr(TypeRepr::LLVMMachine) - } - - fn acle_notation_repr(&self) -> String { - self.repr(TypeRepr::ACLENotation) - } - - fn size(&self) -> String { - self.repr(TypeRepr::Size) - } - - fn size_literal(&self) -> String { - self.repr(TypeRepr::SizeLiteral) - } - - fn type_kind(&self) -> String { - self.repr(TypeRepr::TypeKind) - } - - fn size_in_bytes_log2(&self) -> String { - self.repr(TypeRepr::SizeInBytesLog2) - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] -pub struct TypeKindOptions { - f: bool, - s: bool, - u: bool, - p: bool, -} - -impl TypeKindOptions { - pub fn contains(&self, kind: BaseTypeKind) -> bool { - match kind { - BaseTypeKind::Float => self.f, - BaseTypeKind::Int => self.s, - BaseTypeKind::UInt => self.u, - BaseTypeKind::Poly => self.p, - BaseTypeKind::Bool => false, - } - } -} - -impl FromStr for TypeKindOptions { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut result = Self::default(); - for kind in s.bytes() { - match kind { - b'f' => result.f = true, - b's' => result.s = true, - b'u' => result.u = true, - b'p' => result.p = true, - _ => { - return Err(format!("unknown type kind: {}", char::from(kind))); - } - } - } - Ok(result) - } -} - -impl fmt::Display for TypeKindOptions { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.f.then(|| write!(f, "f")).transpose()?; - self.s.then(|| write!(f, "s")).transpose()?; - self.u.then(|| write!(f, "u")).transpose().map(|_| ()) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum BaseTypeKind { - Float, - Int, - UInt, - Bool, - Poly, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum BaseType { - Sized(BaseTypeKind, u32), - Unsized(BaseTypeKind), -} - -#[derive( - Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, SerializeDisplay, DeserializeFromStr, -)] -pub enum VectorTupleSize { - Two, - Three, - Four, -} - -impl VectorTupleSize { - pub fn to_int(&self) -> u32 { - match self { - Self::Two => 2, - Self::Three => 3, - Self::Four => 4, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct VectorType { - base_type: BaseType, - lanes: u32, - is_scalable: bool, - tuple_size: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] -pub enum TypeKind { - Vector(VectorType), - Base(BaseType), - Pointer(Box, AccessLevel), - Custom(String), - Wildcard(Wildcard), -} - -impl TypeKind { - pub fn base_type(&self) -> Option<&BaseType> { - match self { - Self::Vector(t) => Some(t.base_type()), - Self::Pointer(t, _) => t.base_type(), - Self::Base(t) => Some(t), - Self::Wildcard(..) => None, - Self::Custom(..) => None, - } - } - - pub fn base_type_mut(&mut self) -> Option<&mut BaseType> { - match self { - Self::Vector(t) => Some(t.base_type_mut()), - Self::Pointer(t, _) => t.base_type_mut(), - Self::Base(t) => Some(t), - Self::Wildcard(..) => None, - Self::Custom(..) => None, - } - } - - pub fn populate_wildcard(&mut self, type_kind: TypeKind) -> context::Result { - match self { - Self::Wildcard(..) => *self = type_kind, - Self::Pointer(t, _) => t.populate_wildcard(type_kind)?, - _ => return Err("no wildcard available to populate".to_string()), - } - Ok(()) - } - - pub fn base(&self) -> Option<&BaseType> { - match self { - Self::Base(ty) => Some(ty), - Self::Pointer(tk, _) => tk.base(), - Self::Vector(ty) => Some(&ty.base_type), - _ => None, - } - } - - pub fn vector(&self) -> Option<&VectorType> { - match self { - Self::Vector(ty) => Some(ty), - _ => None, - } - } - - pub fn vector_mut(&mut self) -> Option<&mut VectorType> { - match self { - Self::Vector(ty) => Some(ty), - _ => None, - } - } - - pub fn wildcard(&self) -> Option<&Wildcard> { - match self { - Self::Wildcard(w) => Some(w), - Self::Pointer(w, _) => w.wildcard(), - _ => None, - } - } - - pub fn make_predicate_from(ty: &TypeKind) -> context::Result { - Ok(TypeKind::Vector(VectorType::make_predicate_from_bitsize( - ty.base_type() - .ok_or_else(|| format!("cannot infer predicate from type {ty}"))? - .get_size() - .map_err(|_| format!("cannot infer predicate from unsized type {ty}"))?, - ))) - } - - pub fn make_vector( - from: TypeKind, - is_scalable: bool, - tuple_size: Option, - ) -> context::Result { - from.base().cloned().map_or_else( - || Err(format!("cannot make a vector type out of {from}!")), - |base| { - let vt = VectorType::make_from_base(base, is_scalable, tuple_size); - Ok(TypeKind::Vector(vt)) - }, - ) - } - - /// Return a new expression that converts the provided `expr` from type `other` to `self`. - /// - /// Conversions are bitwise over the whole value, like `transmute`, though `transmute` - /// itself is only used as a last resort. - /// - /// This can fail (returning `None`) due to incompatible types, and many conversions are simply - /// unimplemented. - pub fn express_reinterpretation_from( - &self, - other: &TypeKind, - expr: impl Into, - ) -> Option { - if self == other { - Some(expr.into()) - } else if let (Some(self_vty), Some(other_vty)) = (self.vector(), other.vector()) { - if self_vty.is_scalable - && self_vty.tuple_size.is_none() - && other_vty.is_scalable - && other_vty.tuple_size.is_none() - { - // Plain scalable vectors. - use BaseTypeKind::*; - match (self_vty.base_type, other_vty.base_type) { - (BaseType::Sized(Int, self_size), BaseType::Sized(UInt, other_size)) - if self_size == other_size => - { - Some(Expression::MethodCall( - Box::new(expr.into()), - "as_signed".parse().unwrap(), - vec![], - )) - } - (BaseType::Sized(UInt, self_size), BaseType::Sized(Int, other_size)) - if self_size == other_size => - { - Some(Expression::MethodCall( - Box::new(expr.into()), - "as_unsigned".parse().unwrap(), - vec![], - )) - } - ( - BaseType::Sized(Float | Int | UInt, _), - BaseType::Sized(Float | Int | UInt, _), - ) => Some(FnCall::new_expression( - // Conversions between float and (u)int, or where the lane size changes. - "simd_reinterpret".parse().unwrap(), - vec![expr.into()], - )), - _ => None, - } - } else { - // Tuples and fixed-width vectors. - None - } - } else { - // Scalar types. - None - } - } -} - -impl FromStr for TypeKind { - type Err = String; - - fn from_str(s: &str) -> Result { - Ok(match s { - s if s.starts_with('{') && s.ends_with('}') => { - Self::Wildcard(s[1..s.len() - 1].trim().parse()?) - } - s if s.starts_with('*') => { - let mut split = s[1..].split_whitespace(); - let (ty, rw) = match (split.clone().count(), split.next(), split.next()) { - (2, Some("mut"), Some(ty)) => (ty, AccessLevel::RW), - (2, Some("const"), Some(ty)) => (ty, AccessLevel::R), - (1, Some(ty), None) => (ty, AccessLevel::R), - _ => return Err(format!("invalid pointer type {s:#?} given")), - }; - Self::Pointer(Box::new(ty.parse()?), rw) - } - _ => s - .parse::() - .map(TypeKind::Vector) - .or_else(|_| s.parse::().map(TypeKind::Base)) - .unwrap_or_else(|_| TypeKind::Custom(s.to_string())), - }) - } -} - -impl fmt::Display for TypeKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Vector(ty) => write!(f, "{ty}"), - Self::Pointer(ty, _) => write!(f, "{ty}"), - Self::Base(ty) => write!(f, "{ty}"), - Self::Wildcard(w) => write!(f, "{{{w}}}"), - Self::Custom(s) => write!(f, "{s}"), - } - } -} - -impl ToRepr for TypeKind { - fn repr(&self, repr: TypeRepr) -> String { - match self { - Self::Vector(ty) => ty.repr(repr), - Self::Pointer(ty, _) => ty.repr(repr), - Self::Base(ty) => ty.repr(repr), - Self::Wildcard(w) => format!("{w}"), - Self::Custom(s) => s.to_string(), - } - } -} - -impl ToTokens for TypeKind { - fn to_tokens(&self, tokens: &mut TokenStream) { - if let Self::Pointer(_, rw) = self { - tokens.append_all(match rw { - AccessLevel::RW => quote! { *mut }, - AccessLevel::R => quote! { *const }, - }) - } - - tokens.append_all( - self.to_string() - .parse::() - .expect("invalid syntax"), - ) - } -} - -impl PartialOrd for TypeKind { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl From<&TypeKind> for usize { - fn from(ty: &TypeKind) -> Self { - match ty { - TypeKind::Base(_) => 1, - TypeKind::Pointer(_, _) => 2, - TypeKind::Vector(_) => 3, - TypeKind::Custom(_) => 4, - TypeKind::Wildcard(_) => 5, - } - } -} - -impl Ord for TypeKind { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use std::cmp::Ordering::*; - - let self_int: usize = self.into(); - let other_int: usize = other.into(); - - if self_int == other_int { - match (self, other) { - (TypeKind::Base(ty1), TypeKind::Base(ty2)) => ty1.cmp(ty2), - (TypeKind::Pointer(ty1, _), TypeKind::Pointer(ty2, _)) => ty1.cmp(ty2), - (TypeKind::Vector(vt1), TypeKind::Vector(vt2)) => vt1.cmp(vt2), - (TypeKind::Custom(s1), TypeKind::Custom(s2)) => s1.cmp(s2), - (TypeKind::Wildcard(..), TypeKind::Wildcard(..)) => Equal, - _ => unreachable!(), - } - } else { - self_int.cmp(&other_int) - } - } -} - -impl VectorType { - pub fn base_type(&self) -> &BaseType { - &self.base_type - } - - pub fn base_type_mut(&mut self) -> &mut BaseType { - &mut self.base_type - } - - fn sanitise_lanes( - mut base_type: BaseType, - lanes: Option, - ) -> Result<(BaseType, u32), String> { - let lanes = match (base_type, lanes) { - (BaseType::Sized(BaseTypeKind::Bool, lanes), None) => { - base_type = BaseType::Sized(BaseTypeKind::Bool, VECTOR_FULL_REGISTER_SIZE / lanes); - lanes - } - (BaseType::Unsized(BaseTypeKind::Bool), None) => { - base_type = BaseType::Sized(BaseTypeKind::Bool, 8); - 16 - } - (BaseType::Sized(_, size), None) => VECTOR_FULL_REGISTER_SIZE / size, - (BaseType::Sized(_, size), Some(lanes)) => match size * lanes { - VECTOR_FULL_REGISTER_SIZE | VECTOR_HALF_REGISTER_SIZE => lanes, - _ => return Err("invalid number of lanes".to_string()), - }, - _ => return Err("cannot infer number of lanes".to_string()), - }; - - Ok((base_type, lanes)) - } - - pub fn make_from_base( - base_ty: BaseType, - is_scalable: bool, - tuple_size: Option, - ) -> VectorType { - if is_scalable { - if let BaseType::Sized(BaseTypeKind::Bool, size) = base_ty { - return Self::make_predicate_from_bitsize(size); - } - } - - let (base_type, lanes) = Self::sanitise_lanes(base_ty, None).unwrap(); - - VectorType { - base_type, - lanes, - is_scalable, - tuple_size, - } - } - - pub fn make_predicate_from_bitsize(size: u32) -> VectorType { - VectorType { - base_type: BaseType::Sized(BaseTypeKind::Bool, size), - lanes: (VECTOR_FULL_REGISTER_SIZE / size), - is_scalable: true, - tuple_size: None, - } - } - - pub fn cast_base_type_as(&mut self, ty: BaseType) { - self.base_type = ty - } - - pub fn lanes(&self) -> u32 { - self.lanes - } - - pub fn tuple_size(&self) -> Option { - self.tuple_size - } -} - -impl FromStr for VectorType { - type Err = String; - - fn from_str(s: &str) -> Result { - lazy_static! { - static ref RE: Regex = Regex::new(r"^(?:(?:sv(?P(?:uint|int|bool|float)(?:\d+)?))|(?:(?P(?:uint|int|bool|poly|float)(?:\d+)?)x(?P(?:\d+)?)))(?:x(?P2|3|4))?_t$").unwrap(); - } - - if let Some(c) = RE.captures(s) { - let (base_type, lanes) = Self::sanitise_lanes( - c.name("sv_ty") - .or_else(|| c.name("ty")) - .map(<&str>::from) - .map(BaseType::from_str) - .unwrap()?, - c.name("lanes") - .map(<&str>::from) - .map(u32::from_str) - .transpose() - .unwrap(), - ) - .map_err(|e| format!("invalid {s:#?} vector type: {e}"))?; - - let tuple_size = c - .name("tuple_size") - .map(<&str>::from) - .map(VectorTupleSize::from_str) - .transpose() - .unwrap(); - - let v = Ok(VectorType { - base_type, - is_scalable: c.name("sv_ty").is_some(), - lanes, - tuple_size, - }); - return v; - } else { - Err(format!("invalid vector type {s:#?} given")) - } - } -} - -impl ToRepr for VectorType { - fn repr(&self, repr: TypeRepr) -> String { - let make_llvm_repr = |show_unsigned| { - format!( - "{}v{}{}", - if self.is_scalable { "nx" } else { "" }, - self.lanes * (self.tuple_size.map(usize::from).unwrap_or(1) as u32), - match self.base_type { - BaseType::Sized(BaseTypeKind::UInt, size) if show_unsigned => - format!("u{size}"), - _ => self.base_type.llvm_machine_repr(), - } - ) - }; - - if matches!(repr, TypeRepr::ACLENotation) { - self.base_type.acle_notation_repr() - } else if matches!(repr, TypeRepr::LLVMMachine) { - make_llvm_repr(false) - } else if self.is_scalable { - match (self.base_type, self.lanes, self.tuple_size) { - (BaseType::Sized(BaseTypeKind::Bool, _), 16, _) => "svbool_t".to_string(), - (BaseType::Sized(BaseTypeKind::Bool, _), lanes, _) => format!("svbool{lanes}_t"), - (BaseType::Sized(_, size), lanes, _) - if VECTOR_FULL_REGISTER_SIZE != (size * lanes) => - { - // Special internal type case - make_llvm_repr(true) - } - (ty, _, None) => format!("sv{}_t", ty.c_repr()), - (ty, _, Some(tuple_size)) => format!("sv{}x{tuple_size}_t", ty.c_repr()), - } - } else { - match self.tuple_size { - Some(tuple_size) => format!( - "{}x{}x{}_t", - self.base_type.c_repr(), - self.lanes, - tuple_size - ), - None => format!("{}x{}_t", self.base_type.c_repr(), self.lanes), - } - } - } -} - -impl fmt::Display for VectorType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.c_repr()) - } -} - -impl From for usize { - fn from(t: VectorTupleSize) -> Self { - match t { - VectorTupleSize::Two => 2, - VectorTupleSize::Three => 3, - VectorTupleSize::Four => 4, - } - } -} - -impl FromStr for VectorTupleSize { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "2" => Ok(Self::Two), - "3" => Ok(Self::Three), - "4" => Ok(Self::Four), - _ => Err(format!("invalid vector tuple size `{s}` provided")), - } - } -} - -impl TryFrom for VectorTupleSize { - type Error = String; - - fn try_from(value: usize) -> Result { - match value { - 2 => Ok(Self::Two), - 3 => Ok(Self::Three), - 4 => Ok(Self::Four), - _ => Err(format!("invalid vector tuple size `{value}` provided")), - } - } -} - -impl fmt::Display for VectorTupleSize { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", usize::from(*self)) - } -} - -impl FromStr for BaseTypeKind { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "float" | "f" => Ok(Self::Float), - "int" | "i" => Ok(Self::Int), - "uint" | "u" => Ok(Self::UInt), - "poly" | "p" => Ok(Self::Poly), - "bool" | "b" => Ok(Self::Bool), - _ => Err(format!("no match for {s}")), - } - } -} - -impl ToRepr for BaseTypeKind { - fn repr(&self, repr: TypeRepr) -> String { - match (repr, self) { - (TypeRepr::C, Self::Float) => "float", - (TypeRepr::C, Self::Int) => "int", - (TypeRepr::C, Self::UInt) => "uint", - (TypeRepr::C, Self::Poly) => "poly", - (TypeRepr::Rust | TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::Float) => "f", - (TypeRepr::Rust, Self::Int) | (TypeRepr::LLVMMachine, Self::Int | Self::UInt) => "i", - (TypeRepr::Rust | TypeRepr::ACLENotation, Self::UInt) => "u", - (TypeRepr::Rust | TypeRepr::LLVMMachine | TypeRepr::ACLENotation, Self::Poly) => "p", - (TypeRepr::ACLENotation, Self::Int) => "s", - (TypeRepr::ACLENotation, Self::Bool) => "b", - (_, Self::Bool) => "bool", - _ => { - unreachable!("no base type kind available for representation {repr:?}") - } - } - .to_string() - } -} - -impl fmt::Display for BaseTypeKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.c_repr()) - } -} - -impl BaseType { - pub fn get_size(&self) -> Result { - match self { - Self::Sized(_, size) => Ok(*size), - _ => Err(format!("unexpected invalid base type given {self:#?}")), - } - } - - pub fn kind(&self) -> &BaseTypeKind { - match self { - BaseType::Sized(kind, _) | BaseType::Unsized(kind) => kind, - } - } - - pub fn is_bool(&self) -> bool { - self.kind() == &BaseTypeKind::Bool - } - - pub fn is_float(&self) -> bool { - self.kind() == &BaseTypeKind::Float - } -} - -impl FromStr for BaseType { - type Err = String; - - fn from_str(s: &str) -> Result { - lazy_static! { - static ref RE: Regex = Regex::new(r"^(?P[a-zA-Z]+)(?P\d+)?(_t)?$").unwrap(); - } - - if let Some(c) = RE.captures(s) { - let kind = c["kind"].parse()?; - let size = c - .name("size") - .map(<&str>::from) - .map(u32::from_str) - .transpose() - .unwrap(); - match size { - Some(size) => Ok(Self::Sized(kind, size)), - None => Ok(Self::Unsized(kind)), - } - } else { - Err(format!("failed to parse type `{s}`")) - } - } -} - -impl ToRepr for BaseType { - fn repr(&self, repr: TypeRepr) -> String { - use BaseType::*; - use BaseTypeKind::*; - use TypeRepr::*; - match (self, &repr) { - (Sized(Bool, _) | Unsized(Bool), LLVMMachine) => "i1".to_string(), - (Sized(_, size), SizeLiteral) if *size == 8 => "b".to_string(), - (Sized(_, size), SizeLiteral) if *size == 16 => "h".to_string(), - (Sized(_, size), SizeLiteral) if *size == 32 => "w".to_string(), - (Sized(_, size), SizeLiteral) if *size == 64 => "d".to_string(), - (Sized(_, size), SizeLiteral) if *size == 128 => "q".to_string(), - (_, SizeLiteral) => unreachable!("cannot represent {self:#?} as size literal"), - (Sized(Float, _) | Unsized(Float), TypeKind) => "f".to_string(), - (Sized(Int, _) | Unsized(Int), TypeKind) => "s".to_string(), - (Sized(UInt, _) | Unsized(UInt), TypeKind) => "u".to_string(), - (Sized(_, size), Size) => size.to_string(), - (Sized(_, size), SizeInBytesLog2) => { - assert!(size.is_power_of_two() && *size >= 8); - (size >> 3).trailing_zeros().to_string() - } - (Sized(kind, size), _) => format!("{}{size}", kind.repr(repr)), - (Unsized(kind), _) => kind.repr(repr), - } - } -} - -impl fmt::Display for BaseType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.rust_repr()) - } -} - -#[cfg(test)] -mod tests { - use crate::typekinds::*; - - #[test] - fn test_predicate() { - assert_eq!( - "svbool_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::Bool, 8), - is_scalable: true, - lanes: 16, - tuple_size: None - }) - ); - } - - #[test] - fn test_llvm_internal_predicate() { - assert_eq!( - "svbool4_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::Bool, 32), - is_scalable: true, - lanes: 4, - tuple_size: None - }) - ); - } - - #[test] - fn test_llvm_internal_predicate_llvm() { - assert_eq!( - "svbool4_t".parse::().unwrap().llvm_machine_repr(), - "nxv4i1" - ); - } - - #[test] - fn test_llvm_internal_predicate_acle() { - assert_eq!( - "svbool4_t" - .parse::() - .unwrap() - .acle_notation_repr(), - "b32" - ); - } - - #[test] - fn test_predicate_from_bitsize() { - let pg = VectorType::make_predicate_from_bitsize(32); - assert_eq!(pg.acle_notation_repr(), "b32"); - assert_eq!(pg, "svbool4_t".parse().unwrap()); - assert_eq!(pg.lanes, 4); - assert_eq!(pg.base_type, BaseType::Sized(BaseTypeKind::Bool, 32)); - } - - #[test] - fn test_scalable_single() { - assert_eq!( - "svuint8_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 8), - is_scalable: true, - lanes: 16, - tuple_size: None - }) - ); - } - - #[test] - fn test_scalable_tuple() { - assert_eq!( - "svint64x3_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::Int, 64), - is_scalable: true, - lanes: 2, - tuple_size: Some(VectorTupleSize::Three), - }) - ); - } - - #[test] - fn test_scalable_single_llvm() { - assert_eq!( - "svuint32_t" - .parse::() - .unwrap() - .llvm_machine_repr(), - "nxv4i32" - ); - } - - #[test] - fn test_scalable_tuple_llvm() { - assert_eq!( - "svint32x4_t" - .parse::() - .unwrap() - .llvm_machine_repr(), - "nxv16i32" - ); - } - - #[test] - fn test_vector_single_full() { - assert_eq!( - "uint32x4_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 32), - is_scalable: false, - lanes: 4, - tuple_size: None, - }) - ); - } - - #[test] - fn test_vector_single_half() { - assert_eq!( - "uint32x2_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 32), - is_scalable: false, - lanes: 2, - tuple_size: None, - }) - ); - } - - #[test] - fn test_vector_tuple() { - assert_eq!( - "uint64x2x4_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 64), - is_scalable: false, - lanes: 2, - tuple_size: Some(VectorTupleSize::Four), - }) - ); - } - - #[test] - fn test_const_pointer() { - let p = "*u32".parse::().unwrap(); - assert_eq!( - p, - TypeKind::Pointer( - Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32))), - AccessLevel::R - ) - ); - assert_eq!(p.to_token_stream().to_string(), "* const u32") - } - - #[test] - fn test_mut_pointer() { - let p = "*mut u32".parse::().unwrap(); - assert_eq!( - p, - TypeKind::Pointer( - Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32))), - AccessLevel::RW - ) - ); - assert_eq!(p.to_token_stream().to_string(), "* mut u32") - } - - #[test] - #[should_panic] - fn test_invalid_vector_single() { - assert_eq!( - "uint32x8_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 32), - is_scalable: false, - lanes: 8, - tuple_size: None, - }) - ); - } - - #[test] - #[should_panic] - fn test_invalid_vector_tuple() { - assert_eq!( - "uint32x4x5_t".parse::().unwrap(), - TypeKind::Vector(VectorType { - base_type: BaseType::Sized(BaseTypeKind::UInt, 32), - is_scalable: false, - lanes: 8, - tuple_size: None, // cannot represent - }) - ); - } - - #[test] - fn test_base() { - assert_eq!( - "u32".parse::().unwrap(), - TypeKind::Base(BaseType::Sized(BaseTypeKind::UInt, 32)), - ) - } - - #[test] - fn test_custom() { - assert_eq!( - "svpattern".parse::().unwrap(), - TypeKind::Custom("svpattern".to_string()), - ) - } - - #[test] - fn test_wildcard_type() { - assert_eq!( - "{type}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::Type(None)), - ) - } - - #[test] - fn test_wildcard_typeset() { - assert_eq!( - "{type[0]}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::Type(Some(0))), - ) - } - - #[test] - fn test_wildcard_sve_type() { - assert_eq!( - "{sve_type}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::SVEType(None, None)), - ) - } - - #[test] - fn test_wildcard_sve_typeset() { - assert_eq!( - "{sve_type[0]}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::SVEType(Some(0), None)), - ) - } - - #[test] - fn test_wildcard_sve_tuple_type() { - assert_eq!( - "{sve_type_x2}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::SVEType(None, Some(VectorTupleSize::Two))), - ) - } - - #[test] - fn test_wildcard_sve_tuple_typeset() { - assert_eq!( - "{sve_type_x2[0]}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::SVEType(Some(0), Some(VectorTupleSize::Two))), - ) - } - - #[test] - fn test_wildcard_predicate() { - assert_eq!( - "{predicate}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::Predicate(None)) - ) - } - - #[test] - fn test_wildcard_scale() { - assert_eq!( - "{sve_type as i8}".parse::().unwrap(), - TypeKind::Wildcard(Wildcard::Scale( - Box::new(Wildcard::SVEType(None, None)), - Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::Int, 8))) - )) - ) - } - - #[test] - fn test_size_in_bytes_log2() { - assert_eq!("i8".parse::().unwrap().size_in_bytes_log2(), "0"); - assert_eq!("i16".parse::().unwrap().size_in_bytes_log2(), "1"); - assert_eq!("i32".parse::().unwrap().size_in_bytes_log2(), "2"); - assert_eq!("i64".parse::().unwrap().size_in_bytes_log2(), "3") - } - - #[test] - #[should_panic] - fn test_invalid_size_in_bytes_log2() { - "i9".parse::().unwrap().size_in_bytes_log2(); - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/wildcards.rs b/library/stdarch/crates/stdarch-gen2/src/wildcards.rs deleted file mode 100644 index 25aa8034892..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/wildcards.rs +++ /dev/null @@ -1,197 +0,0 @@ -use lazy_static::lazy_static; -use regex::Regex; -use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::fmt; -use std::str::FromStr; - -use crate::{ - fn_suffix::SuffixKind, - predicate_forms::PredicationMask, - typekinds::{ToRepr, TypeKind, TypeKindOptions, VectorTupleSize}, -}; - -#[derive(Debug, Clone, PartialEq, Eq, Hash, SerializeDisplay, DeserializeFromStr)] -pub enum Wildcard { - Type(Option), - /// NEON type derivated by a base type - NEONType(Option, Option, Option), - /// SVE type derivated by a base type - SVEType(Option, Option), - /// Integer representation of bitsize - Size(Option), - /// Integer representation of bitsize minus one - SizeMinusOne(Option), - /// Literal representation of the bitsize: b(yte), h(half), w(ord) or d(ouble) - SizeLiteral(Option), - /// Literal representation of the type kind: f(loat), s(igned), u(nsigned) - TypeKind(Option, Option), - /// Log2 of the size in bytes - SizeInBytesLog2(Option), - /// Predicate to be inferred from the specified type - Predicate(Option), - /// Predicate to be inferred from the greatest type - MaxPredicate, - - Scale(Box, Box), - - // Other wildcards - LLVMLink, - NVariant, - /// Predicate forms to use and placeholder for a predicate form function name modifier - PredicateForms(PredicationMask), - - /// User-set wildcard through `substitutions` - Custom(String), -} - -impl Wildcard { - pub fn is_nonpredicate_type(&self) -> bool { - matches!( - self, - Wildcard::Type(..) | Wildcard::NEONType(..) | Wildcard::SVEType(..) - ) - } - - pub fn get_typeset_index(&self) -> Option { - match self { - Wildcard::Type(idx) | Wildcard::NEONType(idx, ..) | Wildcard::SVEType(idx, ..) => { - Some(idx.unwrap_or(0)) - } - _ => None, - } - } -} - -impl FromStr for Wildcard { - type Err = String; - - fn from_str(s: &str) -> Result { - lazy_static! { - static ref RE: Regex = Regex::new(r"^(?P\w+?)(?:_x(?P[2-4]))?(?:\[(?P\d+)\])?(?:\.(?P\w+))?(?:\s+as\s+(?P.*?))?$").unwrap(); - } - - if let Some(c) = RE.captures(s) { - let wildcard_name = &c["wildcard"]; - let inputset_index = c - .name("index") - .map(<&str>::from) - .map(usize::from_str) - .transpose() - .map_err(|_| format!("{:#?} is not a valid type index", &c["index"]))?; - let tuple_size = c - .name("tuple_size") - .map(<&str>::from) - .map(VectorTupleSize::from_str) - .transpose() - .map_err(|_| format!("{:#?} is not a valid tuple size", &c["tuple_size"]))?; - let modifiers = c.name("modifiers").map(<&str>::from); - - let wildcard = match (wildcard_name, inputset_index, tuple_size, modifiers) { - ("type", index, None, None) => Ok(Wildcard::Type(index)), - ("neon_type", index, tuple, modifier) => { - if let Some(str_suffix) = modifier { - let suffix_kind = SuffixKind::from_str(str_suffix); - return Ok(Wildcard::NEONType(index, tuple, Some(suffix_kind.unwrap()))); - } else { - Ok(Wildcard::NEONType(index, tuple, None)) - } - } - ("sve_type", index, tuple, None) => Ok(Wildcard::SVEType(index, tuple)), - ("size", index, None, None) => Ok(Wildcard::Size(index)), - ("size_minus_one", index, None, None) => Ok(Wildcard::SizeMinusOne(index)), - ("size_literal", index, None, None) => Ok(Wildcard::SizeLiteral(index)), - ("type_kind", index, None, modifiers) => Ok(Wildcard::TypeKind( - index, - modifiers.map(|modifiers| modifiers.parse()).transpose()?, - )), - ("size_in_bytes_log2", index, None, None) => Ok(Wildcard::SizeInBytesLog2(index)), - ("predicate", index, None, None) => Ok(Wildcard::Predicate(index)), - ("max_predicate", None, None, None) => Ok(Wildcard::MaxPredicate), - ("llvm_link", None, None, None) => Ok(Wildcard::LLVMLink), - ("_n", None, None, None) => Ok(Wildcard::NVariant), - (w, None, None, None) if w.starts_with('_') => { - // test for predicate forms - let pf_mask = PredicationMask::from_str(&w[1..]); - if let Ok(mask) = pf_mask { - if mask.has_merging() { - Ok(Wildcard::PredicateForms(mask)) - } else { - Err("cannot add predication without a Merging form".to_string()) - } - } else { - Err(format!("invalid wildcard `{s:#?}`")) - } - } - (cw, None, None, None) => Ok(Wildcard::Custom(cw.to_string())), - _ => Err(format!("invalid wildcard `{s:#?}`")), - }?; - - let scale_to = c - .name("scale_to") - .map(<&str>::from) - .map(TypeKind::from_str) - .transpose() - .map_err(|_| format!("{:#?} is not a valid type", &c["scale_to"]))?; - - if let Some(scale_to) = scale_to { - Ok(Wildcard::Scale(Box::new(wildcard), Box::new(scale_to))) - } else { - Ok(wildcard) - } - } else { - Err(format!("## invalid wildcard `{s:#?}`")) - } - } -} - -impl fmt::Display for Wildcard { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Type(None) => write!(f, "type"), - Self::Type(Some(index)) => write!(f, "type[{index}]"), - Self::NEONType(None, None, None) => write!(f, "neon_type"), - Self::NEONType(None, None, Some(suffix_kind)) => write!(f, "neon_type.{suffix_kind}"), - Self::NEONType(Some(index), None, None) => write!(f, "neon_type[{index}]"), - Self::NEONType(Some(index), None, Some(suffix_kind)) => { - write!(f, "neon_type[{index}].{suffix_kind}") - } - Self::NEONType(None, Some(tuple_size), Some(suffix_kind)) => { - write!(f, "neon_type_x{tuple_size}.{suffix_kind}") - } - Self::NEONType(None, Some(tuple_size), None) => write!(f, "neon_type_x{tuple_size}"), - Self::NEONType(Some(index), Some(tuple_size), None) => { - write!(f, "neon_type_x{tuple_size}[{index}]") - } - Self::NEONType(Some(index), Some(tuple_size), Some(suffix_kind)) => { - write!(f, "neon_type_x{tuple_size}[{index}].{suffix_kind}") - } - Self::SVEType(None, None) => write!(f, "sve_type"), - Self::SVEType(Some(index), None) => write!(f, "sve_type[{index}]"), - Self::SVEType(None, Some(tuple_size)) => write!(f, "sve_type_x{tuple_size}"), - Self::SVEType(Some(index), Some(tuple_size)) => { - write!(f, "sve_type_x{tuple_size}[{index}]") - } - Self::Size(None) => write!(f, "size"), - Self::Size(Some(index)) => write!(f, "size[{index}]"), - Self::SizeMinusOne(None) => write!(f, "size_minus_one"), - Self::SizeMinusOne(Some(index)) => write!(f, "size_minus_one[{index}]"), - Self::SizeLiteral(None) => write!(f, "size_literal"), - Self::SizeLiteral(Some(index)) => write!(f, "size_literal[{index}]"), - Self::TypeKind(None, None) => write!(f, "type_kind"), - Self::TypeKind(None, Some(opts)) => write!(f, "type_kind.{opts}"), - Self::TypeKind(Some(index), None) => write!(f, "type_kind[{index}]"), - Self::TypeKind(Some(index), Some(opts)) => write!(f, "type_kind[{index}].{opts}"), - Self::SizeInBytesLog2(None) => write!(f, "size_in_bytes_log2"), - Self::SizeInBytesLog2(Some(index)) => write!(f, "size_in_bytes_log2[{index}]"), - Self::Predicate(None) => write!(f, "predicate"), - Self::Predicate(Some(index)) => write!(f, "predicate[{index}]"), - Self::MaxPredicate => write!(f, "max_predicate"), - Self::LLVMLink => write!(f, "llvm_link"), - Self::NVariant => write!(f, "_n"), - Self::PredicateForms(mask) => write!(f, "_{mask}"), - - Self::Scale(wildcard, ty) => write!(f, "{wildcard} as {}", ty.rust_repr()), - Self::Custom(cw) => write!(f, "{cw}"), - } - } -} diff --git a/library/stdarch/crates/stdarch-gen2/src/wildstring.rs b/library/stdarch/crates/stdarch-gen2/src/wildstring.rs deleted file mode 100644 index 2eb467b9628..00000000000 --- a/library/stdarch/crates/stdarch-gen2/src/wildstring.rs +++ /dev/null @@ -1,399 +0,0 @@ -use itertools::Itertools; -use proc_macro2::TokenStream; -use quote::{quote, ToTokens, TokenStreamExt}; -use serde_with::{DeserializeFromStr, SerializeDisplay}; -use std::str::pattern::Pattern; -use std::{fmt, str::FromStr}; - -use crate::context::LocalContext; -use crate::fn_suffix::make_neon_suffix; -use crate::typekinds::{ToRepr, TypeRepr}; -use crate::wildcards::Wildcard; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum WildStringPart { - String(String), - Wildcard(Wildcard), -} - -/// Wildcard-able string -#[derive(Debug, Clone, PartialEq, Eq, Default, SerializeDisplay, DeserializeFromStr)] -pub struct WildString(pub Vec); - -impl WildString { - pub fn has_wildcards(&self) -> bool { - for part in self.0.iter() { - if let WildStringPart::Wildcard(..) = part { - return true; - } - } - - false - } - - pub fn wildcards(&self) -> impl Iterator + '_ { - self.0.iter().filter_map(|part| match part { - WildStringPart::Wildcard(w) => Some(w), - _ => None, - }) - } - - pub fn iter(&self) -> impl Iterator + '_ { - self.0.iter() - } - - pub fn iter_mut(&mut self) -> impl Iterator + '_ { - self.0.iter_mut() - } - - pub fn starts_with(&self, s2: &str) -> bool { - self.to_string().starts_with(s2) - } - - pub fn prepend_str(&mut self, s: impl Into) { - self.0.insert(0, WildStringPart::String(s.into())) - } - - pub fn push_str(&mut self, s: impl Into) { - self.0.push(WildStringPart::String(s.into())) - } - - pub fn push_wildcard(&mut self, w: Wildcard) { - self.0.push(WildStringPart::Wildcard(w)) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn replace<'a, P>(&'a self, from: P, to: &str) -> WildString - where - P: Pattern + Copy, - { - WildString( - self.0 - .iter() - .map(|part| match part { - WildStringPart::String(s) => WildStringPart::String(s.replace(from, to)), - part => part.clone(), - }) - .collect_vec(), - ) - } - - pub fn build_acle(&mut self, ctx: &LocalContext) -> Result<(), String> { - self.build(ctx, TypeRepr::ACLENotation) - } - - pub fn build_neon_intrinsic_signature(&mut self, ctx: &LocalContext) -> Result<(), String> { - let repr = TypeRepr::ACLENotation; - self.iter_mut().try_for_each(|wp| -> Result<(), String> { - if let WildStringPart::Wildcard(w) = wp { - match w { - Wildcard::NEONType(_, _, ref maybe_suffix_kind) => { - if let Some(suffix_kind) = maybe_suffix_kind { - let x = ctx.provide_type_wildcard(w).unwrap(); - *wp = WildStringPart::String(make_neon_suffix(x, *suffix_kind)) - } else { - *wp = WildString::make_default_build(ctx, repr, w) - } - } - _ => *wp = WildString::make_default_build(ctx, repr, w), - } - } - Ok(()) - }) - } - - pub fn build(&mut self, ctx: &LocalContext, repr: TypeRepr) -> Result<(), String> { - match repr { - TypeRepr::ACLENotation | TypeRepr::LLVMMachine => { - self.iter_mut().try_for_each(|wp| -> Result<(), String> { - if let WildStringPart::Wildcard(w) = wp { - match w { - Wildcard::NEONType(_, _, ref maybe_suffix_kind) => { - if let Some(suffix_kind) = maybe_suffix_kind { - let x = ctx.provide_type_wildcard(w).unwrap(); - *wp = WildStringPart::String(make_neon_suffix(x, *suffix_kind)) - } else { - *wp = WildString::make_default_build(ctx, repr, w) - } - } - _ => *wp = WildString::make_default_build(ctx, repr, w), - } - } - Ok(()) - }) - } - _ => self.iter_mut().try_for_each(|wp| -> Result<(), String> { - if let WildStringPart::Wildcard(w) = wp { - *wp = WildString::make_default_build(ctx, repr, w); - } - Ok(()) - }), - } - } - - fn make_default_build(ctx: &LocalContext, repr: TypeRepr, w: &mut Wildcard) -> WildStringPart { - WildStringPart::String( - ctx.provide_substitution_wildcard(w) - .or_else(|_| ctx.provide_type_wildcard(w).map(|ty| ty.repr(repr))) - .unwrap(), - ) - } -} - -impl From for WildString { - fn from(s: String) -> Self { - WildString(vec![WildStringPart::String(s)]) - } -} - -impl FromStr for WildString { - type Err = String; - - fn from_str(s: &str) -> Result { - enum State { - Normal { start: usize }, - Wildcard { start: usize, count: usize }, - EscapeTokenOpen { start: usize, at: usize }, - EscapeTokenClose { start: usize, at: usize }, - } - - let mut ws = WildString::default(); - match s - .char_indices() - .try_fold(State::Normal { start: 0 }, |state, (idx, ch)| { - match (state, ch) { - (State::Normal { start }, '{') => Ok(State::EscapeTokenOpen { start, at: idx }), - (State::Normal { start }, '}') => { - Ok(State::EscapeTokenClose { start, at: idx }) - } - (State::EscapeTokenOpen { start, at }, '{') - | (State::EscapeTokenClose { start, at }, '}') => { - if start < at { - ws.push_str(&s[start..at]) - } - - Ok(State::Normal { start: idx }) - } - (State::EscapeTokenOpen { at, .. }, '}') => Err(format!( - "empty wildcard given in string {s:?} at position {at}" - )), - (State::EscapeTokenOpen { start, at }, _) => { - if start < at { - ws.push_str(&s[start..at]) - } - - Ok(State::Wildcard { - start: idx, - count: 0, - }) - } - (State::EscapeTokenClose { at, .. }, _) => Err(format!( - "closing a non-wildcard/bad escape in string {s:?} at position {at}" - )), - // Nesting wildcards is only supported for `{foo as {bar}}`, wildcards cannot be - // nested at the start of a WildString. - (State::Wildcard { start, count }, '{') => Ok(State::Wildcard { - start, - count: count + 1, - }), - (State::Wildcard { start, count: 0 }, '}') => { - ws.push_wildcard(s[start..idx].parse()?); - Ok(State::Normal { start: idx + 1 }) - } - (State::Wildcard { start, count }, '}') => Ok(State::Wildcard { - start, - count: count - 1, - }), - (state @ State::Normal { .. }, _) | (state @ State::Wildcard { .. }, _) => { - Ok(state) - } - } - })? { - State::Normal { start } => { - if start < s.len() { - ws.push_str(&s[start..]); - } - - Ok(ws) - } - State::EscapeTokenOpen { at, .. } | State::Wildcard { start: at, .. } => Err(format!( - "unclosed wildcard in string {s:?} at position {at}" - )), - State::EscapeTokenClose { at, .. } => Err(format!( - "closing a non-wildcard/bad escape in string {s:?} at position {at}" - )), - } - } -} - -impl fmt::Display for WildString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{}", - self.0 - .iter() - .map(|part| match part { - WildStringPart::String(s) => s.to_owned(), - WildStringPart::Wildcard(w) => format!("{{{w}}}"), - }) - .join("") - ) - } -} - -impl ToTokens for WildString { - fn to_tokens(&self, tokens: &mut TokenStream) { - assert!( - !self.has_wildcards(), - "cannot convert string with wildcards {self:?} to TokenStream" - ); - let str = self.to_string(); - tokens.append_all(quote! { #str }) - } -} - -#[cfg(test)] -mod tests { - use crate::typekinds::*; - use crate::wildstring::*; - - #[test] - fn test_empty_string() { - let ws: WildString = "".parse().unwrap(); - assert_eq!(ws.0.len(), 0); - } - - #[test] - fn test_plain_string() { - let ws: WildString = "plain string".parse().unwrap(); - assert_eq!(ws.0.len(), 1); - assert_eq!( - ws, - WildString(vec![WildStringPart::String("plain string".to_string())]) - ) - } - - #[test] - fn test_escaped_curly_brackets() { - let ws: WildString = "VALUE = {{value}}".parse().unwrap(); - assert_eq!(ws.to_string(), "VALUE = {value}"); - assert!(!ws.has_wildcards()); - } - - #[test] - fn test_escaped_curly_brackets_wildcard() { - let ws: WildString = "TYPE = {{{type}}}".parse().unwrap(); - assert_eq!(ws.to_string(), "TYPE = {{type}}"); - assert_eq!(ws.0.len(), 4); - assert!(ws.has_wildcards()); - } - - #[test] - fn test_wildcard_right_boundary() { - let s = "string test {type}"; - let ws: WildString = s.parse().unwrap(); - assert_eq!(&ws.to_string(), s); - assert!(ws.has_wildcards()); - } - - #[test] - fn test_wildcard_left_boundary() { - let s = "{type} string test"; - let ws: WildString = s.parse().unwrap(); - assert_eq!(&ws.to_string(), s); - assert!(ws.has_wildcards()); - } - - #[test] - fn test_recursive_wildcard() { - let s = "string test {type[0] as {type[1]}}"; - let ws: WildString = s.parse().unwrap(); - - assert_eq!(ws.0.len(), 2); - assert_eq!( - ws, - WildString(vec![ - WildStringPart::String("string test ".to_string()), - WildStringPart::Wildcard(Wildcard::Scale( - Box::new(Wildcard::Type(Some(0))), - Box::new(TypeKind::Wildcard(Wildcard::Type(Some(1)))), - )) - ]) - ); - } - - #[test] - fn test_scale_wildcard() { - let s = "string {type[0] as i8} test"; - let ws: WildString = s.parse().unwrap(); - - assert_eq!(ws.0.len(), 3); - assert_eq!( - ws, - WildString(vec![ - WildStringPart::String("string ".to_string()), - WildStringPart::Wildcard(Wildcard::Scale( - Box::new(Wildcard::Type(Some(0))), - Box::new(TypeKind::Base(BaseType::Sized(BaseTypeKind::Int, 8))), - )), - WildStringPart::String(" test".to_string()) - ]) - ); - } - - #[test] - fn test_solitaire_wildcard() { - let ws: WildString = "{type}".parse().unwrap(); - assert_eq!(ws.0.len(), 1); - assert_eq!( - ws, - WildString(vec![WildStringPart::Wildcard(Wildcard::Type(None))]) - ) - } - - #[test] - fn test_empty_wildcard() { - "string {}" - .parse::() - .expect_err("expected parse error"); - } - - #[test] - fn test_invalid_open_wildcard_right() { - "string {" - .parse::() - .expect_err("expected parse error"); - } - - #[test] - fn test_invalid_close_wildcard_right() { - "string }" - .parse::() - .expect_err("expected parse error"); - } - - #[test] - fn test_invalid_open_wildcard_left() { - "{string" - .parse::() - .expect_err("expected parse error"); - } - - #[test] - fn test_invalid_close_wildcard_left() { - "}string" - .parse::() - .expect_err("expected parse error"); - } - - #[test] - fn test_consecutive_wildcards() { - let s = "svprf{size_literal[1]}_gather_{type[0]}{index_or_offset}"; - let ws: WildString = s.parse().unwrap(); - assert_eq!(ws.to_string(), s) - } -} diff --git a/library/stdarch/crates/stdarch-verify/src/lib.rs b/library/stdarch/crates/stdarch-verify/src/lib.rs index c53f1e62975..5e4db96a70d 100644 --- a/library/stdarch/crates/stdarch-verify/src/lib.rs +++ b/library/stdarch/crates/stdarch-verify/src/lib.rs @@ -45,9 +45,7 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { for &mut (ref mut file, ref path) in &mut files { for mut item in file.items.drain(..) { match item { - syn::Item::Fn(f) => { - functions.push((f, path)); - } + syn::Item::Fn(f) => functions.push((f, path)), syn::Item::Mod(ref mut m) => { if let Some(ref mut m) = m.content { for i in m.1.drain(..) { @@ -73,9 +71,12 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { assert!(!tests.is_empty()); functions.retain(|(f, _)| { - matches!(f.vis, syn::Visibility::Public(_)) - // Many SVE intrinsics are safe - && (f.sig.unsafety.is_some() || f.sig.ident.to_string().starts_with("sv")) + if let syn::Visibility::Public(_) = f.vis { + if f.sig.unsafety.is_some() { + return true; + } + } + false }); assert!(!functions.is_empty()); @@ -98,7 +99,7 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { for generic in f.sig.generics.params.iter() { match *generic { syn::GenericParam::Const(ref c) => const_arguments.push(to_type(&c.ty)), - syn::GenericParam::Type(_) => (), + syn::GenericParam::Type(ref _t) => (), _ => panic!("invalid generic argument on {name}"), }; } @@ -117,31 +118,25 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { }; let required_const = find_required_const("rustc_args_required_const", &f.attrs); - let mut const_generics_indices = + let mut legacy_const_generics = find_required_const("rustc_legacy_const_generics", &f.attrs); - if !required_const.is_empty() && !const_generics_indices.is_empty() { + if !required_const.is_empty() && !legacy_const_generics.is_empty() { panic!( "Can't have both #[rustc_args_required_const] and \ #[rustc_legacy_const_generics]" ); } - // Newer intrinsics don't have legacy support - assume they belong at the end of the argument list - if required_const.is_empty() && const_generics_indices.is_empty() { - const_generics_indices = - (arguments.len()..(arguments.len() + const_arguments.len())).collect(); - } - // The list of required consts, used to verify the arguments, comes from either the // `rustc_args_required_const` or the `rustc_legacy_const_generics` attribute. let required_const = if required_const.is_empty() { - const_generics_indices.clone() + legacy_const_generics.clone() } else { required_const }; - const_generics_indices.sort(); - for (idx, ty) in const_generics_indices + legacy_const_generics.sort(); + for (idx, ty) in legacy_const_generics .into_iter() .zip(const_arguments.into_iter()) { @@ -150,12 +145,12 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream { // strip leading underscore from fn name when building a test // _mm_foo -> mm_foo such that the test name is test_mm_foo. - let test_name = name.to_string(); - let test_name_id = test_name.trim_start_matches('_'); - let has_test = tests.contains(&format!("test_{test_name_id}")) - // SVE load/store tests - || tests.iter().any(|t| t.starts_with(&format!("test_{test_name_id}")) - || t.ends_with(&format!("_with_{test_name_id}"))); + let test_name_string = format!("{name}"); + let mut test_name_id = test_name_string.as_str(); + while test_name_id.starts_with('_') { + test_name_id = &test_name_id[1..]; + } + let has_test = tests.contains(&format!("test_{test_name_id}")); let doc = find_doc(&f.attrs); @@ -226,53 +221,8 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream { "p16" => quote! { &P16 }, "Ordering" => quote! { &ORDERING }, "CpuidResult" => quote! { &CPUID }, - "T" => quote! { &GENERICT }, // arm ... - "svbool_t" => quote! { &SVBOOL }, - "svint8_t" => quote! { &SVI8 }, - "svint8x2_t" => quote! { &SVI8X2 }, - "svint8x3_t" => quote! { &SVI8X3 }, - "svint8x4_t" => quote! { &SVI8X4 }, - "svint16_t" => quote! { &SVI16 }, - "svint16x2_t" => quote! { &SVI16X2 }, - "svint16x3_t" => quote! { &SVI16X3 }, - "svint16x4_t" => quote! { &SVI16X4 }, - "svint32_t" => quote! { &SVI32 }, - "svint32x2_t" => quote! { &SVI32X2 }, - "svint32x3_t" => quote! { &SVI32X3 }, - "svint32x4_t" => quote! { &SVI32X4 }, - "svint64_t" => quote! { &SVI64 }, - "svint64x2_t" => quote! { &SVI64X2 }, - "svint64x3_t" => quote! { &SVI64X3 }, - "svint64x4_t" => quote! { &SVI64X4 }, - "svuint8_t" => quote! { &SVU8 }, - "svuint8x2_t" => quote! { &SVU8X2 }, - "svuint8x3_t" => quote! { &SVU8X3 }, - "svuint8x4_t" => quote! { &SVU8X4 }, - "svuint16_t" => quote! { &SVU16 }, - "svuint16x2_t" => quote! { &SVU16X2 }, - "svuint16x3_t" => quote! { &SVU16X3 }, - "svuint16x4_t" => quote! { &SVU16X4 }, - "svuint32_t" => quote! { &SVU32 }, - "svuint32x2_t" => quote! { &SVU32X2 }, - "svuint32x3_t" => quote! { &SVU32X3 }, - "svuint32x4_t" => quote! { &SVU32X4 }, - "svuint64_t" => quote! { &SVU64 }, - "svuint64x2_t" => quote! { &SVU64X2 }, - "svuint64x3_t" => quote! { &SVU64X3 }, - "svuint64x4_t" => quote! { &SVU64X4 }, - "svfloat32_t" => quote! { &SVF32 }, - "svfloat32x2_t" => quote! { &SVF32X2 }, - "svfloat32x3_t" => quote! { &SVF32X3 }, - "svfloat32x4_t" => quote! { &SVF32X4 }, - "svfloat64_t" => quote! { &SVF64 }, - "svfloat64x2_t" => quote! { &SVF64X2 }, - "svfloat64x3_t" => quote! { &SVF64X3 }, - "svfloat64x4_t" => quote! { &SVF64X4 }, - "svprfop" => quote! { &SVPRFOP }, - "svpattern" => quote! { &SVPATTERN }, - "int8x4_t" => quote! { &I8X4 }, "int8x8_t" => quote! { &I8X8 }, "int8x8x2_t" => quote! { &I8X8X2 }, diff --git a/library/stdarch/crates/stdarch-verify/tests/arm.rs b/library/stdarch/crates/stdarch-verify/tests/arm.rs index fe51a6730cf..a35b8175fb2 100644 --- a/library/stdarch/crates/stdarch-verify/tests/arm.rs +++ b/library/stdarch/crates/stdarch-verify/tests/arm.rs @@ -27,8 +27,6 @@ static U16: Type = Type::PrimUnsigned(16); static U32: Type = Type::PrimUnsigned(32); static U64: Type = Type::PrimUnsigned(64); static U8: Type = Type::PrimUnsigned(8); -static BOOL: Type = Type::PrimBool; -static VOID: Type = Type::Void; static NEVER: Type = Type::Never; static GENERICT: Type = Type::GenericParam("T"); static GENERICU: Type = Type::GenericParam("U"); @@ -153,70 +151,19 @@ static U8X8X2: Type = Type::U(8, 8, 2); static U8X8X3: Type = Type::U(8, 8, 3); static U8X8X4: Type = Type::U(8, 8, 4); -static SVBOOL: Type = Type::Pred; -static SVF32: Type = Type::SVF(32, 1); -static SVF32X2: Type = Type::SVF(32, 2); -static SVF32X3: Type = Type::SVF(32, 3); -static SVF32X4: Type = Type::SVF(32, 4); -static SVF64: Type = Type::SVF(64, 1); -static SVF64X2: Type = Type::SVF(64, 2); -static SVF64X3: Type = Type::SVF(64, 3); -static SVF64X4: Type = Type::SVF(64, 4); -static SVI8: Type = Type::SVI(8, 1); -static SVI8X2: Type = Type::SVI(8, 2); -static SVI8X3: Type = Type::SVI(8, 3); -static SVI8X4: Type = Type::SVI(8, 4); -static SVI16: Type = Type::SVI(16, 1); -static SVI16X2: Type = Type::SVI(16, 2); -static SVI16X3: Type = Type::SVI(16, 3); -static SVI16X4: Type = Type::SVI(16, 4); -static SVI32: Type = Type::SVI(32, 1); -static SVI32X2: Type = Type::SVI(32, 2); -static SVI32X3: Type = Type::SVI(32, 3); -static SVI32X4: Type = Type::SVI(32, 4); -static SVI64: Type = Type::SVI(64, 1); -static SVI64X2: Type = Type::SVI(64, 2); -static SVI64X3: Type = Type::SVI(64, 3); -static SVI64X4: Type = Type::SVI(64, 4); -static SVU8: Type = Type::SVU(8, 1); -static SVU8X2: Type = Type::SVU(8, 2); -static SVU8X3: Type = Type::SVU(8, 3); -static SVU8X4: Type = Type::SVU(8, 4); -static SVU16: Type = Type::SVU(16, 1); -static SVU16X2: Type = Type::SVU(16, 2); -static SVU16X3: Type = Type::SVU(16, 3); -static SVU16X4: Type = Type::SVU(16, 4); -static SVU32: Type = Type::SVU(32, 1); -static SVU32X2: Type = Type::SVU(32, 2); -static SVU32X3: Type = Type::SVU(32, 3); -static SVU32X4: Type = Type::SVU(32, 4); -static SVU64: Type = Type::SVU(64, 1); -static SVU64X2: Type = Type::SVU(64, 2); -static SVU64X3: Type = Type::SVU(64, 3); -static SVU64X4: Type = Type::SVU(64, 4); -static SVPRFOP: Type = Type::Enum("svprfop"); -static SVPATTERN: Type = Type::Enum("svpattern"); - #[derive(Debug, Copy, Clone, PartialEq)] enum Type { - Void, - PrimBool, PrimFloat(u8), PrimSigned(u8), PrimUnsigned(u8), PrimPoly(u8), MutPtr(&'static Type), ConstPtr(&'static Type), - Enum(&'static str), GenericParam(&'static str), I(u8, u8, u8), U(u8, u8, u8), P(u8, u8, u8), F(u8, u8, u8), - Pred, - SVI(u8, u8), - SVU(u8, u8), - SVF(u8, u8), Never, } @@ -235,7 +182,6 @@ fn verify_all_signatures() { let mut all_valid = true; for rust in FUNCTIONS { - // Most SVE intrinsics just rely on the intrinsics test tool for validation if !rust.has_test { let skip = [ "vaddq_s64", @@ -461,18 +407,6 @@ fn verify_all_signatures() { "__clrex", "__dbg", ]; - if !skip.contains(&rust.name) - // Most run-time tests are handled by the intrinsic-test tool, except for - // load/stores (which have generated tests) - && (!rust.name.starts_with("sv") || rust.name.starts_with("svld") - || rust.name.starts_with("svst")) - // The load/store test generator can't handle these cases yet - && (!rust.name.contains("_u32base_") || rust.name.contains("index") || rust.name.contains("offset")) - && !(rust.name.starts_with("svldff1") && rust.name.contains("gather")) - { - println!("missing run-time test for `{}`", rust.name); - all_valid = false; - } } // Skip some intrinsics that aren't NEON and are located in different @@ -545,21 +479,12 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> { let mut nconst = 0; let iter = rust.arguments.iter().zip(&arm.arguments).enumerate(); for (i, (rust_ty, (arm, arm_const))) in iter { - match (*rust_ty, arm) { - // SVE uses generic type parameters to handle void pointers - (Type::ConstPtr(Type::GenericParam("T")), Type::ConstPtr(Type::Void)) => (), - // SVE const generics use i32 over u64 for usability reasons - (Type::PrimSigned(32), Type::PrimUnsigned(64)) if rust.required_const.contains(&i) => { - () - } - // svset doesn't have its const argument last as we assumed when building the Function - _ if rust.name.starts_with("svset") => (), - (x, y) if x == y => (), - _ => bail!("mismatched arguments: {rust_ty:?} != {arm:?}"), + if *rust_ty != arm { + bail!("mismatched arguments: {rust_ty:?} != {arm:?}") } if *arm_const { nconst += 1; - if !rust.required_const.contains(&i) && !rust.name.starts_with("svset") { + if !rust.required_const.contains(&i) { bail!("argument const mismatch"); } } @@ -568,7 +493,7 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> { bail!("wrong number of const arguments"); } - if rust.instrs.is_empty() && arm.instruction != "" { + if rust.instrs.is_empty() { bail!( "instruction not listed for `{}`, but arm lists {:?}", rust.name, @@ -607,7 +532,7 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> { Ok(()) } -#[derive(Debug, PartialEq)] +#[derive(PartialEq)] struct Intrinsic { name: String, ret: Option, @@ -622,7 +547,7 @@ struct JsonIntrinsic { arguments: Vec, return_type: ReturnType, #[serde(default)] - instructions: Option>>, + instructions: Vec>, } #[derive(Deserialize, Debug)] @@ -639,8 +564,8 @@ fn parse_intrinsics(intrinsics: Vec) -> HashMap Intrinsic { - let name = intr.name.replace('[', "").replace(']', ""); +fn parse_intrinsic(mut intr: JsonIntrinsic) -> Intrinsic { + let name = intr.name; let ret = if intr.return_type.value == "void" { None } else { @@ -649,24 +574,18 @@ fn parse_intrinsic(intr: JsonIntrinsic) -> Intrinsic { // This ignores multiple instructions and different optional sequences for now to mimic // the old HTML scraping behaviour - let instruction = intr - .instructions - .map_or(String::new(), |mut i| i.swap_remove(0).swap_remove(0)); + let instruction = intr.instructions.swap_remove(0).swap_remove(0); let arguments = intr .arguments .iter() .map(|s| { - let ty = if let Some(i) = s.find('*') { - &s[..i + 1] - } else { - s.rsplit_once(' ').unwrap().0.trim_start_matches("const ") + let (ty, konst) = match s.strip_prefix("const") { + Some(stripped) => (stripped.trim_start(), true), + None => (s.as_str(), false), }; - let ty = parse_ty(ty); - let konst = s.contains("const") && !matches!(ty, Type::ConstPtr(_)) - || s.starts_with("enum") - || s.rsplit_once(" ").unwrap().1.starts_with("imm"); - (ty, konst) + let ty = ty.rsplit_once(' ').unwrap().0; + (parse_ty(ty), konst) }) .collect::>(); @@ -679,26 +598,18 @@ fn parse_intrinsic(intr: JsonIntrinsic) -> Intrinsic { } fn parse_ty(s: &str) -> Type { - if let Some(ty) = s.strip_suffix("*") { - let ty = ty.trim(); - if let Some(ty) = ty.strip_prefix("const") { - // SVE intrinsics are west-const (const int8_t *) - Type::ConstPtr(parse_ty_base(ty)) - } else if let Some(ty) = ty.strip_suffix("const") { - // Neon intrinsics are east-const (int8_t const *) - Type::ConstPtr(parse_ty_base(ty)) - } else { - Type::MutPtr(parse_ty_base(ty)) - } + let suffix = " const *"; + if let Some(base) = s.strip_suffix(suffix) { + Type::ConstPtr(parse_ty_base(base)) + } else if let Some(base) = s.strip_suffix(" *") { + Type::MutPtr(parse_ty_base(base)) } else { *parse_ty_base(s) } } fn parse_ty_base(s: &str) -> &'static Type { - match s.trim() { - "bool" => &BOOL, - "void" => &VOID, + match s { "float16_t" => &F16, "float16x4_t" => &F16X4, "float16x4x2_t" => &F16X4X2, @@ -828,49 +739,6 @@ fn parse_ty_base(s: &str) -> &'static Type { "uint8x8x2_t" => &U8X8X2, "uint8x8x3_t" => &U8X8X3, "uint8x8x4_t" => &U8X8X4, - "svbool_t" => &SVBOOL, - "svfloat32_t" => &SVF32, - "svfloat32x2_t" => &SVF32X2, - "svfloat32x3_t" => &SVF32X3, - "svfloat32x4_t" => &SVF32X4, - "svfloat64_t" => &SVF64, - "svfloat64x2_t" => &SVF64X2, - "svfloat64x3_t" => &SVF64X3, - "svfloat64x4_t" => &SVF64X4, - "svint8_t" => &SVI8, - "svint8x2_t" => &SVI8X2, - "svint8x3_t" => &SVI8X3, - "svint8x4_t" => &SVI8X4, - "svint16_t" => &SVI16, - "svint16x2_t" => &SVI16X2, - "svint16x3_t" => &SVI16X3, - "svint16x4_t" => &SVI16X4, - "svint32_t" => &SVI32, - "svint32x2_t" => &SVI32X2, - "svint32x3_t" => &SVI32X3, - "svint32x4_t" => &SVI32X4, - "svint64_t" => &SVI64, - "svint64x2_t" => &SVI64X2, - "svint64x3_t" => &SVI64X3, - "svint64x4_t" => &SVI64X4, - "svuint8_t" => &SVU8, - "svuint8x2_t" => &SVU8X2, - "svuint8x3_t" => &SVU8X3, - "svuint8x4_t" => &SVU8X4, - "svuint16_t" => &SVU16, - "svuint16x2_t" => &SVU16X2, - "svuint16x3_t" => &SVU16X3, - "svuint16x4_t" => &SVU16X4, - "svuint32_t" => &SVU32, - "svuint32x2_t" => &SVU32X2, - "svuint32x3_t" => &SVU32X3, - "svuint32x4_t" => &SVU32X4, - "svuint64_t" => &SVU64, - "svuint64x2_t" => &SVU64X2, - "svuint64x3_t" => &SVU64X3, - "svuint64x4_t" => &SVU64X4, - "enum svprfop" => &SVPRFOP, - "enum svpattern" => &SVPATTERN, _ => panic!("failed to parse json type {s:?}"), } -- cgit 1.4.1-3-g733a5