diff options
Diffstat (limited to 'compiler/rustc_codegen_llvm/src/intrinsic.rs')
| -rw-r--r-- | compiler/rustc_codegen_llvm/src/intrinsic.rs | 187 |
1 files changed, 115 insertions, 72 deletions
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index e63fb22829a..f51d014bfb3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -7,7 +7,6 @@ use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; use crate::value::Value; -use rustc_ast as ast; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -19,13 +18,16 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_middle::{bug, span_bug}; use rustc_span::{sym, symbol::kw, Span, Symbol}; -use rustc_target::abi::{self, HasDataLayout, Primitive}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; use rustc_target::spec::{HasTargetSpec, PanicStrategy}; use std::cmp::Ordering; use std::iter; -fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<(&'ll Type, &'ll Value)> { +fn get_simple_intrinsic<'ll>( + cx: &CodegenCx<'ll, '_>, + name: Symbol, +) -> Option<(&'ll Type, &'ll Value)> { let llvm_name = match name { sym::sqrtf32 => "llvm.sqrt.f32", sym::sqrtf64 => "llvm.sqrt.f64", @@ -74,7 +76,7 @@ fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: Symbol) -> Option<(&'ll T Some(cx.get_intrinsic(llvm_name)) } -impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { +impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_intrinsic_call( &mut self, instance: ty::Instance<'tcx>, @@ -348,8 +350,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.type_void(), true, false, - ast::LlvmAsmDialect::Att, + llvm::AsmDialect::Att, &[span], + false, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); @@ -392,15 +396,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) } - fn sideeffect(&mut self) { - // This kind of check would make a ton of sense in the caller, but currently the only - // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM - // codegen backend being used, and so is unable to check the LLVM version. - if unsafe { llvm::LLVMRustVersionMajor() } < 12 { - self.call_intrinsic("llvm.sideeffect", &[]); - } - } - fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time // optimization pass replaces calls to this intrinsic with code to test type membership. @@ -418,8 +413,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } -fn try_intrinsic( - bx: &mut Builder<'a, 'll, 'tcx>, +fn try_intrinsic<'ll>( + bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, @@ -448,8 +443,8 @@ fn try_intrinsic( // instructions are meant to work for all targets, as of the time of this // writing, however, LLVM does not recommend the usage of these new instructions // as the old ones are still more optimized. -fn codegen_msvc_try( - bx: &mut Builder<'a, 'll, 'tcx>, +fn codegen_msvc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, @@ -530,9 +525,8 @@ fn codegen_msvc_try( normal.ret(bx.const_i32(0)); - let cs = catchswitch.catch_switch(None, None, 2); - catchswitch.add_handler(cs, catchpad_rust.llbb()); - catchswitch.add_handler(cs, catchpad_foreign.llbb()); + let cs = + catchswitch.catch_switch(None, None, &[catchpad_rust.llbb(), catchpad_foreign.llbb()]); // We can't use the TypeDescriptor defined in libpanic_unwind because it // might be in another DLL and the SEH encoding only supports specifying @@ -600,8 +594,8 @@ fn codegen_msvc_try( // function calling it, and that function may already have other personality // functions in play. By calling a shim we're guaranteed that our shim will have // the right personality function. -fn codegen_gnu_try( - bx: &mut Builder<'a, 'll, 'tcx>, +fn codegen_gnu_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, @@ -656,8 +650,8 @@ fn codegen_gnu_try( // Variant of codegen_gnu_try used for emscripten where Rust panics are // implemented using C++ exceptions. Here we use exceptions of a specific type // (`struct rust_panic`) to represent Rust panics. -fn codegen_emcc_try( - bx: &mut Builder<'a, 'll, 'tcx>, +fn codegen_emcc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, try_func: &'ll Value, data: &'ll Value, catch_func: &'ll Value, @@ -795,7 +789,7 @@ fn get_rust_try_fn<'ll, 'tcx>( ))); // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( - vec![try_fn_ty, i8p, catch_fn_ty].into_iter(), + [try_fn_ty, i8p, catch_fn_ty].into_iter(), tcx.types.i32, false, hir::Unsafety::Unsafe, @@ -806,8 +800,8 @@ fn get_rust_try_fn<'ll, 'tcx>( rust_try } -fn generic_simd_intrinsic( - bx: &mut Builder<'a, 'll, 'tcx>, +fn generic_simd_intrinsic<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, &'ll Value>], @@ -857,28 +851,39 @@ fn generic_simd_intrinsic( let arg_tys = sig.inputs(); if name == sym::simd_select_bitmask { - let in_ty = arg_tys[0]; - let m_len = match in_ty.kind() { - // Note that this `.unwrap()` crashes for isize/usize, that's sort - // of intentional as there's not currently a use case for that. - ty::Int(i) => i.bit_width().unwrap(), - ty::Uint(i) => i.bit_width().unwrap(), - _ => return_error!("`{}` is not an integral type", in_ty), - }; require_simd!(arg_tys[1], "argument"); - let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); - require!( - // Allow masks for vectors with fewer than 8 elements to be - // represented with a u8 or i8. - m_len == v_len || (m_len == 8 && v_len < 8), - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, - v_len - ); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + + let expected_int_bits = (len.max(8) - 1).next_power_of_two(); + let expected_bytes = len / 8 + ((len % 8 > 0) as u64); + + let mask_ty = arg_tys[0]; + let mask = match mask_ty.kind() { + ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + let place = PlaceRef::alloca(bx, args[0].layout); + args[0].val.store(bx, place); + let int_ty = bx.type_ix(expected_bytes * 8); + let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); + bx.load(int_ty, ptr, Align::ONE) + } + _ => return_error!( + "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + mask_ty, + expected_int_bits, + expected_bytes + ), + }; + let i1 = bx.type_i1(); - let im = bx.type_ix(v_len); - let i1xn = bx.type_vector(i1, v_len); - let m_im = bx.trunc(args[0].immediate(), im); + let im = bx.type_ix(len); + let i1xn = bx.type_vector(i1, len); + let m_im = bx.trunc(mask, im); let m_i1s = bx.bitcast(m_im, i1xn); return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } @@ -1056,16 +1061,16 @@ fn generic_simd_intrinsic( if name == sym::simd_bitmask { // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a - // vector mask and returns an unsigned integer containing the most - // significant bit (MSB) of each lane. - - // If the vector has less than 8 lanes, a u8 is returned with zeroed - // trailing bits. + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. let expected_int_bits = in_len.max(8); - match ret_ty.kind() { - ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (), - _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits), - } + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); // Integer vector <i{in_bitwidth} x in_len>: let (i_xn, in_elem_bitwidth) = match in_elem.kind() { @@ -1095,16 +1100,42 @@ fn generic_simd_intrinsic( let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); // Bitcast <i1 x N> to iN: let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); - // Zero-extend iN to the bitmask type: - return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + + match ret_ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { + // Zero-extend iN to the bitmask type: + return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + } + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + // Zero-extend iN to the array lengh: + let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); + + // Convert the integer to a byte array + let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + bx.store(ze, ptr, Align::ONE); + let array_ty = bx.type_array(bx.type_i8(), expected_bytes); + let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); + return Ok(bx.load(array_ty, ptr, Align::ONE)); + } + _ => return_error!( + "cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret_ty, + expected_int_bits, + expected_bytes + ), + } } - fn simd_simple_float_intrinsic( + fn simd_simple_float_intrinsic<'ll, 'tcx>( name: Symbol, in_elem: &::rustc_middle::ty::TyS<'_>, in_ty: &::rustc_middle::ty::TyS<'_>, in_len: u64, - bx: &mut Builder<'a, 'll, 'tcx>, + bx: &mut Builder<'_, 'll, 'tcx>, span: Span, args: &[OperandRef<'tcx, &'ll Value>], ) -> Result<&'ll Value, ()> { @@ -1202,7 +1233,7 @@ fn generic_simd_intrinsic( elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize, - bx: &Builder<'a, 'll, 'tcx>, + bx: &Builder<'_, '_, '_>, ) -> String { let p0s: String = "p0".repeat(no_pointers); match *elem_ty.kind() { @@ -1225,7 +1256,7 @@ fn generic_simd_intrinsic( } } - fn llvm_vector_ty( + fn llvm_vector_ty<'ll>( cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64, @@ -1656,7 +1687,7 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, bitwise_red!(simd_reduce_all: vector_reduce_and, true); bitwise_red!(simd_reduce_any: vector_reduce_or, true); - if name == sym::simd_cast { + if name == sym::simd_cast || name == sym::simd_as { require_simd!(ret_ty, "return"); let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); require!( @@ -1682,14 +1713,26 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let (in_style, in_width) = match in_elem.kind() { // vectors of pointer-sized integers should've been // disallowed before here, so this unwrap is safe. - ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), - ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), ty::Float(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0), }; let (out_style, out_width) = match out_elem.kind() { - ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), - ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), ty::Float(f) => (Style::Float, f.bit_width()), _ => (Style::Unsupported, 0), }; @@ -1716,10 +1759,10 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, }); } (Style::Float, Style::Int(out_is_signed)) => { - return Ok(if out_is_signed { - bx.fptosi(args[0].immediate(), llret_ty) - } else { - bx.fptoui(args[0].immediate(), llret_ty) + return Ok(match (out_is_signed, name == sym::simd_as) { + (false, false) => bx.fptoui(args[0].immediate(), llret_ty), + (true, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), }); } (Style::Float, Style::Float) => { |