about summary refs log tree commit diff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/intrinsics/simd.rs227
1 files changed, 161 insertions, 66 deletions
diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs
index 292d3ca45f1..644d379d0fa 100644
--- a/src/intrinsics/simd.rs
+++ b/src/intrinsics/simd.rs
@@ -6,16 +6,23 @@ use rustc_span::Symbol;
 use super::*;
 use crate::prelude::*;
 
-fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span: Span, ty: Ty<'_>) {
-    if !ty.is_simd() {
-        fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
-        // Prevent verifier error
-        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+fn report_simd_type_validation_error(
+    fx: &mut FunctionCx<'_, '_, '_>,
+    intrinsic: Symbol,
+    span: Span,
+    ty: Ty<'_>,
+) {
+    fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
+    // Prevent verifier error
+    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+}
+
+macro simd_cmp($fx:expr, $intrinsic:ident, $span:ident, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
         return;
     }
-}
 
-macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
         let res_lane = match lane_ty.kind() {
@@ -32,7 +39,12 @@ macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident)
     });
 }
 
-macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_int_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -43,7 +55,12 @@ macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $r
     });
 }
 
-macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_int_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -55,7 +72,12 @@ macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident,
     });
 }
 
-macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+macro simd_flt_binop($fx:expr, $intrinsic:ident, $span:ident, $op:ident($x:ident, $y:ident) -> $ret:ident) {
+    if !$x.layout().ty.is_simd() {
+        report_simd_type_validation_error($fx, $intrinsic, $span, $x.layout().ty);
+        return;
+    }
+
     // FIXME use vector instructions when possible
     simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
         match lane_ty.kind() {
@@ -80,7 +102,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_cast, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
                 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
 
@@ -92,29 +118,25 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_eq, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, Equal|Equal|Equal(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, Equal|Equal|Equal(x, y) -> ret);
         };
         simd_ne, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, NotEqual|NotEqual|NotEqual(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, NotEqual|NotEqual|NotEqual(x, y) -> ret);
         };
         simd_lt, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
         };
         simd_le, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
         };
         simd_gt, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
+            simd_cmp!(fx, intrinsic, span, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
         };
         simd_ge, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
             simd_cmp!(
                 fx,
+                intrinsic,
+                span,
                 UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
                 (x, y) -> ret
             );
@@ -122,7 +144,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
         // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
         _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
 
             // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
             // If there is no suffix, use the index array length.
@@ -224,7 +249,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_extract, (c v, o idx) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                 idx_const
             } else {
@@ -252,7 +281,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_neg, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                 match lane_ty.kind() {
                     ty::Int(_) => fx.bcx.ins().ineg(lane),
@@ -263,37 +296,45 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fabs, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().fabs(lane)
             });
         };
 
         simd_fsqrt, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().sqrt(lane)
             });
         };
 
         simd_add, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, iadd|iadd|fadd(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, iadd|iadd|fadd(x, y) -> ret);
         };
         simd_sub, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, isub|isub|fsub(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, isub|isub|fsub(x, y) -> ret);
         };
         simd_mul, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, imul|imul|fmul(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, imul|imul|fmul(x, y) -> ret);
         };
         simd_div, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
+            simd_int_flt_binop!(fx, intrinsic, span, udiv|sdiv|fdiv(x, y) -> ret);
         };
         simd_rem, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
+            if !x.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
+                return;
+            }
+
             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
                 match lane_ty.kind() {
                     ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
@@ -315,28 +356,26 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_shl, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, ishl|ishl(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, ishl|ishl(x, y) -> ret);
         };
         simd_shr, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, ushr|sshr(x, y) -> ret);
         };
         simd_and, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, band|band(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, band|band(x, y) -> ret);
         };
         simd_or, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bor|bor(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, bor|bor(x, y) -> ret);
         };
         simd_xor, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_int_binop!(fx, bxor|bxor(x, y) -> ret);
+            simd_int_binop!(fx, intrinsic, span, bxor|bxor(x, y) -> ret);
         };
 
         simd_fma, (c a, c b, c c) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
             assert_eq!(a.layout(), b.layout());
             assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
@@ -359,16 +398,18 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_fmin, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_flt_binop!(fx, fmin(x, y) -> ret);
+            simd_flt_binop!(fx, intrinsic, span, fmin(x, y) -> ret);
         };
         simd_fmax, (c x, c y) {
-            validate_simd_type(fx, intrinsic, span, x.layout().ty);
-            simd_flt_binop!(fx, fmax(x, y) -> ret);
+            simd_flt_binop!(fx, intrinsic, span, fmax(x, y) -> ret);
         };
 
         simd_round, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                 match lane_ty.kind() {
                     ty::Float(FloatTy::F32) => fx.lib_call(
@@ -388,26 +429,42 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         };
         simd_ceil, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().ceil(lane)
             });
         };
         simd_floor, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().floor(lane)
             });
         };
         simd_trunc, (c a) {
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
+
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
                 fx.bcx.ins().trunc(lane)
             });
         };
 
         simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                 if lane_ty.is_floating_point() {
                     fx.bcx.ins().fadd(a, b)
@@ -418,7 +475,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                 if lane_ty.is_floating_point() {
                     fx.bcx.ins().fmul(a, b)
@@ -429,32 +490,56 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_all, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_any, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_and, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
         };
 
         simd_reduce_or, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
         };
 
         simd_reduce_xor, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
         };
 
         simd_reduce_min, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                 let lt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
@@ -467,7 +552,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_reduce_max, (c v) {
-            validate_simd_type(fx, intrinsic, span, v.layout().ty);
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
             simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                 let gt = match ty.kind() {
                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
@@ -480,8 +569,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         };
 
         simd_select, (c m, c a, c b) {
-            validate_simd_type(fx, intrinsic, span, m.layout().ty);
-            validate_simd_type(fx, intrinsic, span, a.layout().ty);
+            if !m.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
+                return;
+            }
+            if !a.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+                return;
+            }
             assert_eq!(a.layout(), b.layout());
 
             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
generated by cgit-pink 1.4.1-3-g733a5 (git 2.37.1) at 2026-04-23 22:35:33 +0000