diff options
Diffstat (limited to 'compiler/rustc_codegen_gcc/src/intrinsic/simd.rs')
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/intrinsic/simd.rs | 466 |
1 files changed, 237 insertions, 229 deletions
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs index 12a7dab155b..84cd5b002fb 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs @@ -62,11 +62,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument { - span, - name, - ty: arg_tys[1] - }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let expected_int_bits = (len.max(8) - 1).next_power_of_two(); @@ -140,14 +139,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); require!( bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } @@ -269,23 +271,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( let lo_nibble = bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &lo_nibble_elements); - let mask = bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &vec![ - bx.context - .new_rvalue_from_int( - bx.u8_type, 0x0f - ); - byte_vector_type_size - as _ - ]); - - let four_vec = bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &vec![ - bx.context - .new_rvalue_from_int( - bx.u8_type, 4 - ); - byte_vector_type_size - as _ - ]); + let mask = bx.context.new_rvalue_from_vector( + None, + long_byte_vector_type, + &vec![bx.context.new_rvalue_from_int(bx.u8_type, 0x0f); byte_vector_type_size as _], + ); + + let four_vec = bx.context.new_rvalue_from_vector( + None, + long_byte_vector_type, + &vec![bx.context.new_rvalue_from_int(bx.u8_type, 4); byte_vector_type_size as _], + ); // Step 2: Byte-swap the input. let swapped = simd_bswap(bx, args[0].immediate()); @@ -388,21 +384,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); - require!(out_len == n, InvalidMonomorphization::ReturnLength { - span, - name, - in_len: n, - ret_ty, - out_len - }); - require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement { - span, - name, - in_elem, - in_ty, - ret_ty, - out_ty - }); + require!( + out_len == n, + InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } + ); + require!( + in_elem == out_ty, + InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } + ); let vector = args[2].immediate(); @@ -411,13 +400,16 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( #[cfg(feature = "master")] if name == sym::simd_insert { - require!(in_elem == arg_tys[2], InvalidMonomorphization::InsertedType { - span, - name, - in_elem, - in_ty, - out_ty: arg_tys[2] - }); + require!( + in_elem == arg_tys[2], + InvalidMonomorphization::InsertedType { + span, + name, + in_elem, + in_ty, + out_ty: arg_tys[2] + } + ); let vector = args[0].immediate(); let index = args[1].immediate(); let value = args[2].immediate(); @@ -431,13 +423,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( #[cfg(feature = "master")] if name == sym::simd_extract { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); let vector = args[0].immediate(); return Ok(bx.context.new_vector_access(None, vector, args[1].immediate()).to_rvalue()); } @@ -445,18 +434,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( if name == sym::simd_select { let m_elem_ty = in_elem; let m_len = in_len; - require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument { - span, - name, - ty: arg_tys[1] - }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } + ); let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!(m_len == v_len, InvalidMonomorphization::MismatchedLengths { - span, - name, - m_len, - v_len - }); + require!( + m_len == v_len, + InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } + ); match *m_elem_ty.kind() { ty::Int(_) => {} _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), @@ -468,25 +454,27 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match *in_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { - span, - name, - ty: in_elem - }); + require!( + metadata.is_unit(), + InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem } + ); } _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) @@ -497,11 +485,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty) }); - require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer { - span, - name, - ty: out_elem - }); + require!( + metadata.is_unit(), + InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem } + ); } _ => { return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) @@ -524,14 +511,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match *in_elem.kind() { ty::RawPtr(_, _) => {} @@ -560,14 +550,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); match *in_elem.kind() { ty::Uint(ty::UintTy::Usize) => {} @@ -596,14 +589,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( if name == sym::simd_cast || name == sym::simd_as { require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType { - span, - name, - in_len, - in_ty, - ret_ty, - out_len - }); + require!( + in_len == out_len, + InvalidMonomorphization::ReturnLengthInputType { + span, + name, + in_len, + in_ty, + ret_ty, + out_len + } + ); // casting cares about nominal type, not just structural type if in_elem == out_elem { return Ok(args[0].immediate()); @@ -629,14 +625,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( match (in_style, out_style) { (Style::Unsupported, Style::Unsupported) => { - require!(false, InvalidMonomorphization::UnsupportedCast { - span, - name, - in_ty, - in_elem, - ret_ty, - out_elem - }); + require!( + false, + InvalidMonomorphization::UnsupportedCast { + span, + name, + in_ty, + in_elem, + ret_ty, + out_elem + } + ); } _ => return Ok(bx.context.convert_vector(None, args[0].immediate(), llret_ty)), } @@ -914,45 +913,47 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( // All types must be simd vector types require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); - require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond { - span, - name, - ty: arg_tys[1] - }); - require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird { - span, - name, - ty: arg_tys[2] - }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); // Of the same length: let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); - require!(in_len == out_len, InvalidMonomorphization::SecondArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[1], - out_len - }); - require!(in_len == out_len2, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[2], - out_len: out_len2 - }); + require!( + in_len == out_len, + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len + } + ); + require!( + in_len == out_len2, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: out_len2 + } + ); // The return type must match the first argument type - require!(ret_ty == in_ty, InvalidMonomorphization::ExpectedReturnType { - span, - name, - in_ty, - ret_ty - }); + require!( + ret_ty == in_ty, + InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } + ); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -979,15 +980,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( (ptr_count(element_ty1), non_ptr(element_ty1)) } _ => { - require!(false, InvalidMonomorphization::ExpectedElementType { - span, - name, - expected_element: element_ty1, - second_arg: arg_tys[1], - in_elem, - in_ty, - mutability: ExpectedPointerMutability::Not, - }); + require!( + false, + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Not, + } + ); unreachable!(); } }; @@ -1000,12 +1004,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( match *element_ty2.kind() { ty::Int(_) => (), _ => { - require!(false, InvalidMonomorphization::ThirdArgElementType { - span, - name, - expected_element: element_ty2, - third_arg: arg_tys[2] - }); + require!( + false, + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } + ); } } @@ -1029,36 +1036,40 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( // All types must be simd vector types require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); - require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond { - span, - name, - ty: arg_tys[1] - }); - require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird { - span, - name, - ty: arg_tys[2] - }); + require_simd!( + arg_tys[1], + InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } + ); + require_simd!( + arg_tys[2], + InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } + ); // Of the same length: let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); - require!(in_len == element_len1, InvalidMonomorphization::SecondArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[1], - out_len: element_len1 - }); - require!(in_len == element_len2, InvalidMonomorphization::ThirdArgumentLength { - span, - name, - in_len, - in_ty, - arg_ty: arg_tys[2], - out_len: element_len2 - }); + require!( + in_len == element_len1, + InvalidMonomorphization::SecondArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[1], + out_len: element_len1 + } + ); + require!( + in_len == element_len2, + InvalidMonomorphization::ThirdArgumentLength { + span, + name, + in_len, + in_ty, + arg_ty: arg_tys[2], + out_len: element_len2 + } + ); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -1086,15 +1097,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( (ptr_count(element_ty1), non_ptr(element_ty1)) } _ => { - require!(false, InvalidMonomorphization::ExpectedElementType { - span, - name, - expected_element: element_ty1, - second_arg: arg_tys[1], - in_elem, - in_ty, - mutability: ExpectedPointerMutability::Mut, - }); + require!( + false, + InvalidMonomorphization::ExpectedElementType { + span, + name, + expected_element: element_ty1, + second_arg: arg_tys[1], + in_elem, + in_ty, + mutability: ExpectedPointerMutability::Mut, + } + ); unreachable!(); } }; @@ -1106,12 +1120,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( match *element_ty2.kind() { ty::Int(_) => (), _ => { - require!(false, InvalidMonomorphization::ThirdArgElementType { - span, - name, - expected_element: element_ty2, - third_arg: arg_tys[2] - }); + require!( + false, + InvalidMonomorphization::ThirdArgElementType { + span, + name, + expected_element: element_ty2, + third_arg: arg_tys[2] + } + ); } } @@ -1278,13 +1295,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident, $identity:expr) => { if name == sym::$name { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); return match *in_elem.kind() { ty::Int(_) | ty::Uint(_) => { let r = bx.vector_reduce_op(args[0].immediate(), $vec_op); @@ -1350,13 +1364,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( macro_rules! minmax_red { ($name:ident: $int_red:ident, $float_red:ident) => { if name == sym::$name { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); return match *in_elem.kind() { ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())), ty::Float(_) => Ok(bx.$float_red(args[0].immediate())), @@ -1380,13 +1391,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( ($name:ident : $op:expr, $boolean:expr) => { if name == sym::$name { let input = if !$boolean { - require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType { - span, - name, - in_elem, - in_ty, - ret_ty - }); + require!( + ret_ty == in_elem, + InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } + ); args[0].immediate() } else { match *in_elem.kind() { |