diff options
Diffstat (limited to 'src/librustc_codegen_llvm/intrinsic.rs')
| -rw-r--r-- | src/librustc_codegen_llvm/intrinsic.rs | 1288 |
1 files changed, 721 insertions, 567 deletions
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index e5a7583a8cc..abb17b3b0f1 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -1,24 +1,24 @@ -use crate::llvm; -use crate::llvm_util; use crate::abi::{Abi, FnAbi, LlvmType, PassMode}; +use crate::builder::Builder; use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm_util; use crate::type_::Type; use crate::type_of::LayoutLlvmExt; -use crate::builder::Builder; -use crate::value::Value; use crate::va_arg::emit_va_arg; -use rustc_codegen_ssa::MemFlags; -use rustc_codegen_ssa::mir::place::PlaceRef; -use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; -use rustc_codegen_ssa::glue; -use rustc_codegen_ssa::base::{to_immediate, wants_msvc_seh, compare_simd_types}; +use crate::value::Value; +use rustc::hir; +use rustc::ty::layout::{self, FnAbiExt, HasTyCtxt, LayoutOf, Primitive}; use rustc::ty::{self, Ty}; -use rustc::ty::layout::{self, FnAbiExt, LayoutOf, HasTyCtxt, Primitive}; +use rustc::{bug, span_bug}; +use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; -use rustc::hir; +use rustc_codegen_ssa::glue; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::MemFlags; use rustc_target::abi::HasDataLayout; use syntax::ast; -use rustc::{bug, span_bug}; use rustc_codegen_ssa::common::span_invalid_monomorphization_error; use rustc_codegen_ssa::traits::*; @@ -26,7 +26,7 @@ use rustc_codegen_ssa::traits::*; use syntax_pos::Span; use std::cmp::Ordering; -use std::{iter, i128, u128}; +use std::{i128, iter, u128}; fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Value> { let llvm_name = match name { @@ -74,7 +74,7 @@ fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Valu "roundf64" => "llvm.round.f64", "assume" => "llvm.assume", "abort" => "llvm.trap", - _ => return None + _ => return None, }; Some(cx.get_intrinsic(&llvm_name)) } @@ -93,7 +93,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let (def_id, substs) = match callee_ty.kind { ty::FnDef(def_id, substs) => (def_id, substs), - _ => bug!("expected fn item type, found {}", callee_ty) + _ => bug!("expected fn item type, found {}", callee_ty), }; let sig = callee_ty.fn_sig(tcx); @@ -107,14 +107,14 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { let simple = get_simple_intrinsic(self, name); let llval = match name { - _ if simple.is_some() => { - self.call(simple.unwrap(), - &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), - None) - } + _ if simple.is_some() => self.call( + simple.unwrap(), + &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), + None, + ), "unreachable" => { return; - }, + } "likely" => { let expect = self.get_intrinsic(&("llvm.expect.i1")); self.call(expect, &[args[0].immediate(), self.const_bool(true)], None) @@ -124,23 +124,21 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.call(expect, &[args[0].immediate(), self.const_bool(false)], None) } "try" => { - try_intrinsic(self, - args[0].immediate(), - args[1].immediate(), - args[2].immediate(), - llresult); + try_intrinsic( + self, + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + llresult, + ); return; } "breakpoint" => { let llfn = self.get_intrinsic(&("llvm.debugtrap")); self.call(llfn, &[], None) } - "va_start" => { - self.va_start(args[0].immediate()) - } - "va_end" => { - self.va_end(args[0].immediate()) - } + "va_start" => self.va_start(args[0].immediate()), + "va_end" => self.va_end(args[0].immediate()), "va_copy" => { let intrinsic = self.cx().get_intrinsic(&("llvm.va_copy")); self.call(intrinsic, &[args[0].immediate(), args[1].immediate()], None) @@ -155,26 +153,20 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { // less than 4 bytes in length. If it is, promote // the integer to a `i32` and truncate the result // back to the smaller type. - let promoted_result = emit_va_arg(self, args[0], - tcx.types.i32); + let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); self.trunc(promoted_result, llret_ty) } else { emit_va_arg(self, args[0], ret_ty) } } - Primitive::F64 | - Primitive::Pointer => { + Primitive::F64 | Primitive::Pointer => { emit_va_arg(self, args[0], ret_ty) } // `va_arg` should never be used with the return type f32. - Primitive::F32 => { - bug!("the va_arg intrinsic does not work with `f32`") - } + Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), } } - _ => { - bug!("the va_arg intrinsic does not work with non-scalar types") - } + _ => bug!("the va_arg intrinsic does not work with non-scalar types"), } } "size_of_val" => { @@ -195,13 +187,10 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.const_usize(self.align_of(tp_ty).bytes()) } } - "size_of" | - "pref_align_of" | - "min_align_of" | - "needs_drop" | - "type_id" | - "type_name" => { - let ty_name = self.tcx + "size_of" | "pref_align_of" | "min_align_of" | "needs_drop" | "type_id" + | "type_name" => { + let ty_name = self + .tcx .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) .unwrap(); OperandRef::from_const(self, ty_name).immediate_or_packed_pair(self) @@ -219,7 +208,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { ty, llresult, self.const_u8(0), - self.const_usize(1) + self.const_usize(1), ); } return; @@ -240,34 +229,74 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } "copy_nonoverlapping" => { - copy_intrinsic(self, false, false, substs.type_at(0), - args[1].immediate(), args[0].immediate(), args[2].immediate()); + copy_intrinsic( + self, + false, + false, + substs.type_at(0), + args[1].immediate(), + args[0].immediate(), + args[2].immediate(), + ); return; } "copy" => { - copy_intrinsic(self, true, false, substs.type_at(0), - args[1].immediate(), args[0].immediate(), args[2].immediate()); + copy_intrinsic( + self, + true, + false, + substs.type_at(0), + args[1].immediate(), + args[0].immediate(), + args[2].immediate(), + ); return; } "write_bytes" => { - memset_intrinsic(self, false, substs.type_at(0), - args[0].immediate(), args[1].immediate(), args[2].immediate()); + memset_intrinsic( + self, + false, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); return; } "volatile_copy_nonoverlapping_memory" => { - copy_intrinsic(self, false, true, substs.type_at(0), - args[0].immediate(), args[1].immediate(), args[2].immediate()); + copy_intrinsic( + self, + false, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); return; } "volatile_copy_memory" => { - copy_intrinsic(self, true, true, substs.type_at(0), - args[0].immediate(), args[1].immediate(), args[2].immediate()); + copy_intrinsic( + self, + true, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); return; } "volatile_set_memory" => { - memset_intrinsic(self, true, substs.type_at(0), - args[0].immediate(), args[1].immediate(), args[2].immediate()); + memset_intrinsic( + self, + true, + substs.type_at(0), + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ); return; } "volatile_load" | "unaligned_volatile_load" => { @@ -286,243 +315,265 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { llvm::LLVMSetAlignment(load, align); } to_immediate(self, load, self.layout_of(tp_ty)) - }, + } "volatile_store" => { let dst = args[0].deref(self.cx()); args[1].val.volatile_store(self, dst); return; - }, + } "unaligned_volatile_store" => { let dst = args[0].deref(self.cx()); args[1].val.unaligned_volatile_store(self, dst); return; - }, - "prefetch_read_data" | "prefetch_write_data" | - "prefetch_read_instruction" | "prefetch_write_instruction" => { + } + "prefetch_read_data" + | "prefetch_write_data" + | "prefetch_read_instruction" + | "prefetch_write_instruction" => { let expect = self.get_intrinsic(&("llvm.prefetch")); let (rw, cache_type) = match name { "prefetch_read_data" => (0, 1), "prefetch_write_data" => (1, 1), "prefetch_read_instruction" => (0, 0), "prefetch_write_instruction" => (1, 0), - _ => bug!() + _ => bug!(), }; - self.call(expect, &[ - args[0].immediate(), - self.const_i32(rw), - args[1].immediate(), - self.const_i32(cache_type) - ], None) - }, - "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" | - "bitreverse" | "add_with_overflow" | "sub_with_overflow" | - "mul_with_overflow" | "wrapping_add" | "wrapping_sub" | "wrapping_mul" | - "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | - "unchecked_add" | "unchecked_sub" | "unchecked_mul" | "exact_div" | - "rotate_left" | "rotate_right" | "saturating_add" | "saturating_sub" => { + self.call( + expect, + &[ + args[0].immediate(), + self.const_i32(rw), + args[1].immediate(), + self.const_i32(cache_type), + ], + None, + ) + } + "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" + | "bitreverse" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" + | "wrapping_add" | "wrapping_sub" | "wrapping_mul" | "unchecked_div" + | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "unchecked_add" + | "unchecked_sub" | "unchecked_mul" | "exact_div" | "rotate_left" | "rotate_right" + | "saturating_add" | "saturating_sub" => { let ty = arg_tys[0]; match int_type_width_signed(ty, self) { - Some((width, signed)) => - match name { - "ctlz" | "cttz" => { - let y = self.const_bool(false); - let llfn = self.get_intrinsic( - &format!("llvm.{}.i{}", name, width), - ); - self.call(llfn, &[args[0].immediate(), y], None) - } - "ctlz_nonzero" | "cttz_nonzero" => { - let y = self.const_bool(true); - let llvm_name = &format!("llvm.{}.i{}", &name[..4], width); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[args[0].immediate(), y], None) - } - "ctpop" => self.call( - self.get_intrinsic(&format!("llvm.ctpop.i{}", width)), - &[args[0].immediate()], - None - ), - "bswap" => { - if width == 8 { - args[0].immediate() // byte swap a u8/i8 is just a no-op - } else { - self.call( - self.get_intrinsic( - &format!("llvm.bswap.i{}", width), - ), - &[args[0].immediate()], - None, - ) - } - } - "bitreverse" => { + Some((width, signed)) => match name { + "ctlz" | "cttz" => { + let y = self.const_bool(false); + let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width)); + self.call(llfn, &[args[0].immediate(), y], None) + } + "ctlz_nonzero" | "cttz_nonzero" => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.{}.i{}", &name[..4], width); + let llfn = self.get_intrinsic(llvm_name); + self.call(llfn, &[args[0].immediate(), y], None) + } + "ctpop" => self.call( + self.get_intrinsic(&format!("llvm.ctpop.i{}", width)), + &[args[0].immediate()], + None, + ), + "bswap" => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { self.call( - self.get_intrinsic( - &format!("llvm.bitreverse.i{}", width), - ), + self.get_intrinsic(&format!("llvm.bswap.i{}", width)), &[args[0].immediate()], None, ) } - "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => { - let intrinsic = format!("llvm.{}{}.with.overflow.i{}", - if signed { 's' } else { 'u' }, - &name[..3], width); - let llfn = self.get_intrinsic(&intrinsic); - - // Convert `i1` to a `bool`, and write it to the out parameter - let pair = self.call(llfn, &[ - args[0].immediate(), - args[1].immediate() - ], None); + } + "bitreverse" => self.call( + self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)), + &[args[0].immediate()], + None, + ), + "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => { + let intrinsic = format!( + "llvm.{}{}.with.overflow.i{}", + if signed { 's' } else { 'u' }, + &name[..3], + width + ); + let llfn = self.get_intrinsic(&intrinsic); + + // Convert `i1` to a `bool`, and write it to the out parameter + let pair = + self.call(llfn, &[args[0].immediate(), args[1].immediate()], None); + let val = self.extract_value(pair, 0); + let overflow = self.extract_value(pair, 1); + let overflow = self.zext(overflow, self.type_bool()); + + let dest = result.project_field(self, 0); + self.store(val, dest.llval, dest.align); + let dest = result.project_field(self, 1); + self.store(overflow, dest.llval, dest.align); + + return; + } + "wrapping_add" => self.add(args[0].immediate(), args[1].immediate()), + "wrapping_sub" => self.sub(args[0].immediate(), args[1].immediate()), + "wrapping_mul" => self.mul(args[0].immediate(), args[1].immediate()), + "exact_div" => { + if signed { + self.exactsdiv(args[0].immediate(), args[1].immediate()) + } else { + self.exactudiv(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_div" => { + if signed { + self.sdiv(args[0].immediate(), args[1].immediate()) + } else { + self.udiv(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_rem" => { + if signed { + self.srem(args[0].immediate(), args[1].immediate()) + } else { + self.urem(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_shl" => self.shl(args[0].immediate(), args[1].immediate()), + "unchecked_shr" => { + if signed { + self.ashr(args[0].immediate(), args[1].immediate()) + } else { + self.lshr(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_add" => { + if signed { + self.unchecked_sadd(args[0].immediate(), args[1].immediate()) + } else { + self.unchecked_uadd(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_sub" => { + if signed { + self.unchecked_ssub(args[0].immediate(), args[1].immediate()) + } else { + self.unchecked_usub(args[0].immediate(), args[1].immediate()) + } + } + "unchecked_mul" => { + if signed { + self.unchecked_smul(args[0].immediate(), args[1].immediate()) + } else { + self.unchecked_umul(args[0].immediate(), args[1].immediate()) + } + } + "rotate_left" | "rotate_right" => { + let is_left = name == "rotate_left"; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + let llfn = self.get_intrinsic(llvm_name); + self.call(llfn, &[val, val, raw_shift], None) + } + "saturating_add" | "saturating_sub" => { + let is_add = name == "saturating_add"; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + if llvm_util::get_major_version() >= 8 { + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + let llfn = self.get_intrinsic(llvm_name); + self.call(llfn, &[lhs, rhs], None) + } else { + let llvm_name = &format!( + "llvm.{}{}.with.overflow.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + let llfn = self.get_intrinsic(llvm_name); + let pair = self.call(llfn, &[lhs, rhs], None); let val = self.extract_value(pair, 0); let overflow = self.extract_value(pair, 1); - let overflow = self.zext(overflow, self.type_bool()); - - let dest = result.project_field(self, 0); - self.store(val, dest.llval, dest.align); - let dest = result.project_field(self, 1); - self.store(overflow, dest.llval, dest.align); - - return; - }, - "wrapping_add" => self.add(args[0].immediate(), args[1].immediate()), - "wrapping_sub" => self.sub(args[0].immediate(), args[1].immediate()), - "wrapping_mul" => self.mul(args[0].immediate(), args[1].immediate()), - "exact_div" => - if signed { - self.exactsdiv(args[0].immediate(), args[1].immediate()) - } else { - self.exactudiv(args[0].immediate(), args[1].immediate()) - }, - "unchecked_div" => - if signed { - self.sdiv(args[0].immediate(), args[1].immediate()) - } else { - self.udiv(args[0].immediate(), args[1].immediate()) - }, - "unchecked_rem" => - if signed { - self.srem(args[0].immediate(), args[1].immediate()) - } else { - self.urem(args[0].immediate(), args[1].immediate()) - }, - "unchecked_shl" => self.shl(args[0].immediate(), args[1].immediate()), - "unchecked_shr" => - if signed { - self.ashr(args[0].immediate(), args[1].immediate()) - } else { - self.lshr(args[0].immediate(), args[1].immediate()) - }, - "unchecked_add" => { - if signed { - self.unchecked_sadd(args[0].immediate(), args[1].immediate()) - } else { - self.unchecked_uadd(args[0].immediate(), args[1].immediate()) - } - }, - "unchecked_sub" => { - if signed { - self.unchecked_ssub(args[0].immediate(), args[1].immediate()) + let llty = self.type_ix(width); + + let limit = if signed { + let limit_lo = self + .const_uint_big(llty, (i128::MIN >> (128 - width)) as u128); + let limit_hi = self + .const_uint_big(llty, (i128::MAX >> (128 - width)) as u128); + let neg = self.icmp( + IntPredicate::IntSLT, + val, + self.const_uint(llty, 0), + ); + self.select(neg, limit_hi, limit_lo) + } else if is_add { + self.const_uint_big(llty, u128::MAX >> (128 - width)) } else { - self.unchecked_usub(args[0].immediate(), args[1].immediate()) - } - }, - "unchecked_mul" => { - if signed { - self.unchecked_smul(args[0].immediate(), args[1].immediate()) - } else { - self.unchecked_umul(args[0].immediate(), args[1].immediate()) - } - }, - "rotate_left" | "rotate_right" => { - let is_left = name == "rotate_left"; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same - let llvm_name = &format!("llvm.fsh{}.i{}", - if is_left { 'l' } else { 'r' }, width); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[val, val, raw_shift], None) - }, - "saturating_add" | "saturating_sub" => { - let is_add = name == "saturating_add"; - let lhs = args[0].immediate(); - let rhs = args[1].immediate(); - if llvm_util::get_major_version() >= 8 { - let llvm_name = &format!("llvm.{}{}.sat.i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - width); - let llfn = self.get_intrinsic(llvm_name); - self.call(llfn, &[lhs, rhs], None) - } else { - let llvm_name = &format!("llvm.{}{}.with.overflow.i{}", - if signed { 's' } else { 'u' }, - if is_add { "add" } else { "sub" }, - width); - let llfn = self.get_intrinsic(llvm_name); - let pair = self.call(llfn, &[lhs, rhs], None); - let val = self.extract_value(pair, 0); - let overflow = self.extract_value(pair, 1); - let llty = self.type_ix(width); - - let limit = if signed { - let limit_lo = self.const_uint_big( - llty, (i128::MIN >> (128 - width)) as u128); - let limit_hi = self.const_uint_big( - llty, (i128::MAX >> (128 - width)) as u128); - let neg = self.icmp( - IntPredicate::IntSLT, val, self.const_uint(llty, 0)); - self.select(neg, limit_hi, limit_lo) - } else if is_add { - self.const_uint_big(llty, u128::MAX >> (128 - width)) - } else { - self.const_uint(llty, 0) - }; - self.select(overflow, limit, val) - } - }, - _ => bug!(), - }, + self.const_uint(llty, 0) + }; + self.select(overflow, limit, val) + } + } + _ => bug!(), + }, None => { span_invalid_monomorphization_error( - tcx.sess, span, - &format!("invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", name, ty)); + tcx.sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); return; } } - - }, + } "fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" => { match float_type_width(arg_tys[0]) { - Some(_width) => - match name { - "fadd_fast" => self.fadd_fast(args[0].immediate(), args[1].immediate()), - "fsub_fast" => self.fsub_fast(args[0].immediate(), args[1].immediate()), - "fmul_fast" => self.fmul_fast(args[0].immediate(), args[1].immediate()), - "fdiv_fast" => self.fdiv_fast(args[0].immediate(), args[1].immediate()), - "frem_fast" => self.frem_fast(args[0].immediate(), args[1].immediate()), - _ => bug!(), - }, + Some(_width) => match name { + "fadd_fast" => self.fadd_fast(args[0].immediate(), args[1].immediate()), + "fsub_fast" => self.fsub_fast(args[0].immediate(), args[1].immediate()), + "fmul_fast" => self.fmul_fast(args[0].immediate(), args[1].immediate()), + "fdiv_fast" => self.fdiv_fast(args[0].immediate(), args[1].immediate()), + "frem_fast" => self.frem_fast(args[0].immediate(), args[1].immediate()), + _ => bug!(), + }, None => { span_invalid_monomorphization_error( - tcx.sess, span, - &format!("invalid monomorphization of `{}` intrinsic: \ - expected basic float type, found `{}`", name, arg_tys[0])); + tcx.sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic float type, found `{}`", + name, arg_tys[0] + ), + ); return; } } - }, + } "float_to_int_approx_unchecked" => { if float_type_width(arg_tys[0]).is_none() { span_invalid_monomorphization_error( - tcx.sess, span, - &format!("invalid monomorphization of `float_to_int_approx_unchecked` \ + tcx.sess, + span, + &format!( + "invalid monomorphization of `float_to_int_approx_unchecked` \ intrinsic: expected basic float type, \ - found `{}`", arg_tys[0])); + found `{}`", + arg_tys[0] + ), + ); return; } match int_type_width_signed(ret_ty, self.cx) { @@ -535,35 +586,33 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } None => { span_invalid_monomorphization_error( - tcx.sess, span, - &format!("invalid monomorphization of `float_to_int_approx_unchecked` \ + tcx.sess, + span, + &format!( + "invalid monomorphization of `float_to_int_approx_unchecked` \ intrinsic: expected basic integer type, \ - found `{}`", ret_ty)); + found `{}`", + ret_ty + ), + ); return; } } } - "discriminant_value" => { - args[0].deref(self.cx()).codegen_get_discr(self, ret_ty) - } + "discriminant_value" => args[0].deref(self.cx()).codegen_get_discr(self, ret_ty), name if name.starts_with("simd_") => { - match generic_simd_intrinsic(self, name, - callee_ty, - args, - ret_ty, llret_ty, - span) { + match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, - Err(()) => return + Err(()) => return, } } // This requires that atomic intrinsics follow a specific naming pattern: // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst name if name.starts_with("atomic_") => { use rustc_codegen_ssa::common::AtomicOrdering::*; - use rustc_codegen_ssa::common:: - {SynchronizationScope, AtomicRmwBinOp}; + use rustc_codegen_ssa::common::{AtomicRmwBinOp, SynchronizationScope}; let split: Vec<&str> = name.split('_').collect(); @@ -573,29 +622,31 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { 3 => match split[2] { "unordered" => (Unordered, Unordered), "relaxed" => (Monotonic, Monotonic), - "acq" => (Acquire, Acquire), - "rel" => (Release, Monotonic), - "acqrel" => (AcquireRelease, Acquire), - "failrelaxed" if is_cxchg => - (SequentiallyConsistent, Monotonic), - "failacq" if is_cxchg => - (SequentiallyConsistent, Acquire), - _ => self.sess().fatal("unknown ordering in atomic intrinsic") + "acq" => (Acquire, Acquire), + "rel" => (Release, Monotonic), + "acqrel" => (AcquireRelease, Acquire), + "failrelaxed" if is_cxchg => (SequentiallyConsistent, Monotonic), + "failacq" if is_cxchg => (SequentiallyConsistent, Acquire), + _ => self.sess().fatal("unknown ordering in atomic intrinsic"), }, 4 => match (split[2], split[3]) { - ("acq", "failrelaxed") if is_cxchg => - (Acquire, Monotonic), - ("acqrel", "failrelaxed") if is_cxchg => - (AcquireRelease, Monotonic), - _ => self.sess().fatal("unknown ordering in atomic intrinsic") + ("acq", "failrelaxed") if is_cxchg => (Acquire, Monotonic), + ("acqrel", "failrelaxed") if is_cxchg => (AcquireRelease, Monotonic), + _ => self.sess().fatal("unknown ordering in atomic intrinsic"), }, _ => self.sess().fatal("Atomic intrinsic not in correct format"), }; let invalid_monomorphization = |ty| { - span_invalid_monomorphization_error(tcx.sess, span, - &format!("invalid monomorphization of `{}` intrinsic: \ - expected basic integer type, found `{}`", name, ty)); + span_invalid_monomorphization_error( + tcx.sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); }; match split[1] { @@ -609,7 +660,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { args[2].immediate(), order, failorder, - weak); + weak, + ); let val = self.extract_value(pair, 0); let success = self.extract_value(pair, 1); let success = self.zext(success, self.type_bool()); @@ -642,7 +694,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { args[1].immediate(), args[0].immediate(), order, - size + size, ); return; } else { @@ -663,18 +715,18 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { // These are all AtomicRMW ops op => { let atom_op = match op { - "xchg" => AtomicRmwBinOp::AtomicXchg, - "xadd" => AtomicRmwBinOp::AtomicAdd, - "xsub" => AtomicRmwBinOp::AtomicSub, - "and" => AtomicRmwBinOp::AtomicAnd, - "nand" => AtomicRmwBinOp::AtomicNand, - "or" => AtomicRmwBinOp::AtomicOr, - "xor" => AtomicRmwBinOp::AtomicXor, - "max" => AtomicRmwBinOp::AtomicMax, - "min" => AtomicRmwBinOp::AtomicMin, - "umax" => AtomicRmwBinOp::AtomicUMax, - "umin" => AtomicRmwBinOp::AtomicUMin, - _ => self.sess().fatal("unknown atomic operation") + "xchg" => AtomicRmwBinOp::AtomicXchg, + "xadd" => AtomicRmwBinOp::AtomicAdd, + "xsub" => AtomicRmwBinOp::AtomicSub, + "and" => AtomicRmwBinOp::AtomicAnd, + "nand" => AtomicRmwBinOp::AtomicNand, + "or" => AtomicRmwBinOp::AtomicOr, + "xor" => AtomicRmwBinOp::AtomicXor, + "max" => AtomicRmwBinOp::AtomicMax, + "min" => AtomicRmwBinOp::AtomicMin, + "umax" => AtomicRmwBinOp::AtomicUMax, + "umin" => AtomicRmwBinOp::AtomicUMin, + _ => self.sess().fatal("unknown atomic operation"), }; let ty = substs.type_at(0); @@ -683,7 +735,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { atom_op, args[0].immediate(), args[1].immediate(), - order + order, ) } else { return invalid_monomorphization(ty); @@ -725,7 +777,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { self.store(llval, ptr, result.align); } else { OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) - .val.store(self, result); + .val + .store(self, result); } } } @@ -774,11 +827,7 @@ fn copy_intrinsic( ) { let (size, align) = bx.size_and_align_of(ty); let size = bx.mul(bx.const_usize(size.bytes()), count); - let flags = if volatile { - MemFlags::VOLATILE - } else { - MemFlags::empty() - }; + let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; if allow_overlap { bx.memmove(dst, align, src, align, size, flags); } else { @@ -792,15 +841,11 @@ fn memset_intrinsic( ty: Ty<'tcx>, dst: &'ll Value, val: &'ll Value, - count: &'ll Value + count: &'ll Value, ) { let (size, align) = bx.size_and_align_of(ty); let size = bx.mul(bx.const_usize(size.bytes()), count); - let flags = if volatile { - MemFlags::VOLATILE - } else { - MemFlags::empty() - }; + let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; bx.memset(dst, val, size, align, flags); } @@ -1014,7 +1059,7 @@ fn gen_fn<'ll, 'tcx>( output, false, hir::Unsafety::Unsafe, - Abi::Rust + Abi::Rust, )); let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]); let llfn = cx.declare_fn(name, &fn_abi); @@ -1045,7 +1090,7 @@ fn get_rust_try_fn<'ll, 'tcx>( tcx.mk_unit(), false, hir::Unsafety::Unsafe, - Abi::Rust + Abi::Rust, ))); let output = tcx.types.i32; let rust_try = gen_fn(cx, "__rust_try", vec![fn_ty, i8p, i8p], output, codegen); @@ -1060,7 +1105,7 @@ fn generic_simd_intrinsic( args: &[OperandRef<'tcx, &'ll Value>], ret_ty: Ty<'tcx>, llret_ty: &'ll Type, - span: Span + span: Span, ) -> Result<&'ll Value, ()> { // macros for error handling: macro_rules! emit_error { @@ -1095,14 +1140,12 @@ fn generic_simd_intrinsic( macro_rules! require_simd { ($ty: expr, $position: expr) => { require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) - } + }; } let tcx = bx.tcx(); - let sig = tcx.normalize_erasing_late_bound_regions( - ty::ParamEnv::reveal_all(), - &callee_ty.fn_sig(tcx), - ); + let sig = tcx + .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &callee_ty.fn_sig(tcx)); let arg_tys = sig.inputs(); if name == "simd_select_bitmask" { @@ -1116,9 +1159,11 @@ fn generic_simd_intrinsic( }; require_simd!(arg_tys[1], "argument"); let v_len = arg_tys[1].simd_size(tcx); - require!(m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, v_len + require!( + m_len == v_len, + "mismatched lengths: mask length `{}` != other vector length `{}`", + m_len, + v_len ); let i1 = bx.type_i1(); let i1xn = bx.type_vector(i1, m_len); @@ -1139,46 +1184,63 @@ fn generic_simd_intrinsic( "simd_le" => Some(hir::BinOpKind::Le), "simd_gt" => Some(hir::BinOpKind::Gt), "simd_ge" => Some(hir::BinOpKind::Ge), - _ => None + _ => None, }; if let Some(cmp_op) = comparison { require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); - require!(in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ found `{}` with length {}", - in_len, in_ty, - ret_ty, out_len); - require!(bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, - "expected return type with integer elements, found `{}` with non-integer `{}`", - ret_ty, - ret_ty.simd_type(tcx)); - - return Ok(compare_simd_types(bx, - args[0].immediate(), - args[1].immediate(), - in_elem, - llret_ty, - cmp_op)) + in_len, + in_ty, + ret_ty, + out_len + ); + require!( + bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, + "expected return type with integer elements, found `{}` with non-integer `{}`", + ret_ty, + ret_ty.simd_type(tcx) + ); + + return Ok(compare_simd_types( + bx, + args[0].immediate(), + args[1].immediate(), + in_elem, + llret_ty, + cmp_op, + )); } if name.starts_with("simd_shuffle") { - let n: u64 = name["simd_shuffle".len()..].parse().unwrap_or_else(|_| - span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")); + let n: u64 = name["simd_shuffle".len()..].parse().unwrap_or_else(|_| { + span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") + }); require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); - require!(out_len == n, - "expected return type of length {}, found `{}` with length {}", - n, ret_ty, out_len); - require!(in_elem == ret_ty.simd_type(tcx), - "expected return element type `{}` (element of input `{}`), \ + require!( + out_len == n, + "expected return type of length {}, found `{}` with length {}", + n, + ret_ty, + out_len + ); + require!( + in_elem == ret_ty.simd_type(tcx), + "expected return element type `{}` (element of input `{}`), \ found `{}` with element type `{}`", - in_elem, in_ty, - ret_ty, ret_ty.simd_type(tcx)); + in_elem, + in_ty, + ret_ty, + ret_ty.simd_type(tcx) + ); let total_len = u128::from(in_len) * 2; @@ -1194,8 +1256,11 @@ fn generic_simd_intrinsic( None } Some(idx) if idx >= total_len => { - emit_error!("shuffle index #{} is out of bounds (limit {})", - arg_idx, total_len); + emit_error!( + "shuffle index #{} is out of bounds (limit {})", + arg_idx, + total_len + ); None } Some(idx) => Some(bx.const_i32(idx as i32)), @@ -1204,27 +1269,39 @@ fn generic_simd_intrinsic( .collect(); let indices = match indices { Some(i) => i, - None => return Ok(bx.const_null(llret_ty)) + None => return Ok(bx.const_null(llret_ty)), }; - return Ok(bx.shuffle_vector(args[0].immediate(), - args[1].immediate(), - bx.const_vector(&indices))) + return Ok(bx.shuffle_vector( + args[0].immediate(), + args[1].immediate(), + bx.const_vector(&indices), + )); } if name == "simd_insert" { - require!(in_elem == arg_tys[2], - "expected inserted type `{}` (element of input `{}`), found `{}`", - in_elem, in_ty, arg_tys[2]); - return Ok(bx.insert_element(args[0].immediate(), - args[2].immediate(), - args[1].immediate())) + require!( + in_elem == arg_tys[2], + "expected inserted type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + arg_tys[2] + ); + return Ok(bx.insert_element( + args[0].immediate(), + args[2].immediate(), + args[1].immediate(), + )); } if name == "simd_extract" { - require!(ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, in_ty, ret_ty); - return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())) + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); } if name == "simd_select" { @@ -1232,13 +1309,15 @@ fn generic_simd_intrinsic( let m_len = in_len; require_simd!(arg_tys[1], "argument"); let v_len = arg_tys[1].simd_size(tcx); - require!(m_len == v_len, - "mismatched lengths: mask length `{}` != other vector length `{}`", - m_len, v_len + require!( + m_len == v_len, + "mismatched lengths: mask length `{}` != other vector length `{}`", + m_len, + v_len ); match m_elem_ty.kind { - ty::Int(_) => {}, - _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty) + ty::Int(_) => {} + _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), } // truncate the mask to a vector of i1s let i1 = bx.type_i1(); @@ -1256,34 +1335,33 @@ fn generic_simd_intrinsic( // trailing bits. let expected_int_bits = in_len.max(8); match ret_ty.kind { - ty::Uint(i) if i.bit_width() == Some(expected_int_bits as usize) => (), - _ => return_error!( - "bitmask `{}`, expected `u{}`", - ret_ty, expected_int_bits - ), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits as usize) => (), + _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits), } // Integer vector <i{in_bitwidth} x in_len>: let (i_xn, in_elem_bitwidth) = match in_elem.kind { ty::Int(i) => ( args[0].immediate(), - i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _) + i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _), ), ty::Uint(i) => ( args[0].immediate(), - i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _) + i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits() as _), ), _ => return_error!( "vector argument `{}`'s element type `{}`, expected integer element type", - in_ty, in_elem + in_ty, + in_elem ), }; // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. - let shift_indices = vec![ - bx.cx.const_int(bx.type_ix(in_elem_bitwidth as _), (in_elem_bitwidth - 1) as _); - in_len as _ - ]; + let shift_indices = + vec![ + bx.cx.const_int(bx.type_ix(in_elem_bitwidth as _), (in_elem_bitwidth - 1) as _); + in_len as _ + ]; let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); // Truncate vector to an <i1 x N> let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len as _)); @@ -1327,22 +1405,30 @@ fn generic_simd_intrinsic( return_error!( "unsupported floating-point vector `{}` with length `{}` \ out-of-range [2, 16]", - in_ty, in_len); + in_ty, + in_len + ); } "f32" - }, + } ty::Float(f) if f.bit_width() == 64 => { if in_len < 2 || in_len > 8 { - return_error!("unsupported floating-point vector `{}` with length `{}` \ + return_error!( + "unsupported floating-point vector `{}` with length `{}` \ out-of-range [2, 8]", - in_ty, in_len); + in_ty, + in_len + ); } "f64" - }, + } ty::Float(f) => { - return_error!("unsupported element type `{}` of floating-point vector `{}`", - f.name_str(), in_ty); - }, + return_error!( + "unsupported element type `{}` of floating-point vector `{}`", + f.name_str(), + in_ty + ); + } _ => { return_error!("`{}` is not a floating-point type", in_ty); } @@ -1350,9 +1436,8 @@ fn generic_simd_intrinsic( let llvm_name = &format!("llvm.{0}.v{1}{2}", name, in_len, ety); let intrinsic = bx.get_intrinsic(&llvm_name); - let c = bx.call(intrinsic, - &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), - None); + let c = + bx.call(intrinsic, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) }; Ok(c) } @@ -1416,13 +1501,17 @@ fn generic_simd_intrinsic( } } - fn llvm_vector_ty(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64, - mut no_pointers: usize) -> &'ll Type { + fn llvm_vector_ty( + cx: &CodegenCx<'ll, '_>, + elem_ty: Ty<'_>, + vec_len: u64, + mut no_pointers: usize, + ) -> &'ll Type { // FIXME: use cx.layout_of(ty).llvm_type() ? let mut elem_ty = match elem_ty.kind { - ty::Int(v) => cx.type_int_from_ty( v), - ty::Uint(v) => cx.type_uint_from_ty( v), - ty::Float(v) => cx.type_float_from_ty( v), + ty::Int(v) => cx.type_int_from_ty(v), + ty::Uint(v) => cx.type_uint_from_ty(v), + ty::Float(v) => cx.type_float_from_ty(v), _ => unreachable!(), }; while no_pointers > 0 { @@ -1432,7 +1521,6 @@ fn generic_simd_intrinsic( cx.type_vector(elem_ty, vec_len) } - if name == "simd_gather" { // simd_gather(values: <N x T>, pointers: <N x *_ T>, // mask: <N x i{M}>) -> <N x T> @@ -1447,19 +1535,29 @@ fn generic_simd_intrinsic( require_simd!(ret_ty, "return"); // Of the same length: - require!(in_len == arg_tys[1].simd_size(tcx), - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", "second", in_len, in_ty, arg_tys[1], - arg_tys[1].simd_size(tcx)); - require!(in_len == arg_tys[2].simd_size(tcx), - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", "third", in_len, in_ty, arg_tys[2], - arg_tys[2].simd_size(tcx)); + require!( + in_len == arg_tys[1].simd_size(tcx), + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + arg_tys[1].simd_size(tcx) + ); + require!( + in_len == arg_tys[2].simd_size(tcx), + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + arg_tys[2].simd_size(tcx) + ); // The return type must match the first argument type - require!(ret_ty == in_ty, - "expected return type `{}`, found `{}`", - in_ty, ret_ty); + require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -1480,14 +1578,22 @@ fn generic_simd_intrinsic( // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind { - ty::RawPtr(p) if p.ty == in_elem => (ptr_count(arg_tys[1].simd_type(tcx)), - non_ptr(arg_tys[1].simd_type(tcx))), + ty::RawPtr(p) if p.ty == in_elem => { + (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) + } _ => { - require!(false, "expected element type `{}` of second argument `{}` \ + require!( + false, + "expected element type `{}` of second argument `{}` \ to be a pointer to the element type `{}` of the first \ argument `{}`, found `{}` != `*_ {}`", - arg_tys[1].simd_type(tcx), arg_tys[1], in_elem, in_ty, - arg_tys[1].simd_type(tcx), in_elem); + arg_tys[1].simd_type(tcx), + arg_tys[1], + in_elem, + in_ty, + arg_tys[1].simd_type(tcx), + in_elem + ); unreachable!(); } }; @@ -1499,9 +1605,13 @@ fn generic_simd_intrinsic( match arg_tys[2].simd_type(tcx).kind { ty::Int(_) => (), _ => { - require!(false, "expected element type `{}` of third argument `{}` \ + require!( + false, + "expected element type `{}` of third argument `{}` \ to be a signed integer type", - arg_tys[2].simd_type(tcx), arg_tys[2]); + arg_tys[2].simd_type(tcx), + arg_tys[2] + ); } } @@ -1524,17 +1634,17 @@ fn generic_simd_intrinsic( let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); - let llvm_intrinsic = format!("llvm.masked.gather.{}.{}", - llvm_elem_vec_str, llvm_pointer_vec_str); - let f = bx.declare_cfn(&llvm_intrinsic, - bx.type_func(&[ - llvm_pointer_vec_ty, - alignment_ty, - mask_ty, - llvm_elem_vec_ty], llvm_elem_vec_ty)); + let llvm_intrinsic = + format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let f = bx.declare_cfn( + &llvm_intrinsic, + bx.type_func( + &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], + llvm_elem_vec_ty, + ), + ); llvm::SetUnnamedAddr(f, false); - let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], - None); + let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); return Ok(v); } @@ -1551,14 +1661,26 @@ fn generic_simd_intrinsic( require_simd!(arg_tys[2], "third"); // Of the same length: - require!(in_len == arg_tys[1].simd_size(tcx), - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", "second", in_len, in_ty, arg_tys[1], - arg_tys[1].simd_size(tcx)); - require!(in_len == arg_tys[2].simd_size(tcx), - "expected {} argument with length {} (same as input type `{}`), \ - found `{}` with length {}", "third", in_len, in_ty, arg_tys[2], - arg_tys[2].simd_size(tcx)); + require!( + in_len == arg_tys[1].simd_size(tcx), + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + arg_tys[1].simd_size(tcx) + ); + require!( + in_len == arg_tys[2].simd_size(tcx), + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + arg_tys[2].simd_size(tcx) + ); // This counts how many pointers fn ptr_count(t: Ty<'_>) -> usize { @@ -1579,15 +1701,22 @@ fn generic_simd_intrinsic( // The second argument must be a simd vector with an element type that's a pointer // to the element type of the first argument let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind { - ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut - => (ptr_count(arg_tys[1].simd_type(tcx)), - non_ptr(arg_tys[1].simd_type(tcx))), + ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { + (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) + } _ => { - require!(false, "expected element type `{}` of second argument `{}` \ + require!( + false, + "expected element type `{}` of second argument `{}` \ to be a pointer to the element type `{}` of the first \ argument `{}`, found `{}` != `*mut {}`", - arg_tys[1].simd_type(tcx), arg_tys[1], in_elem, in_ty, - arg_tys[1].simd_type(tcx), in_elem); + arg_tys[1].simd_type(tcx), + arg_tys[1], + in_elem, + in_ty, + arg_tys[1].simd_type(tcx), + in_elem + ); unreachable!(); } }; @@ -1599,9 +1728,13 @@ fn generic_simd_intrinsic( match arg_tys[2].simd_type(tcx).kind { ty::Int(_) => (), _ => { - require!(false, "expected element type `{}` of third argument `{}` \ + require!( + false, + "expected element type `{}` of third argument `{}` \ to be a signed integer type", - arg_tys[2].simd_type(tcx), arg_tys[2]); + arg_tys[2].simd_type(tcx), + arg_tys[2] + ); } } @@ -1626,25 +1759,27 @@ fn generic_simd_intrinsic( let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); - let llvm_intrinsic = format!("llvm.masked.scatter.{}.{}", - llvm_elem_vec_str, llvm_pointer_vec_str); - let f = bx.declare_cfn(&llvm_intrinsic, - bx.type_func(&[llvm_elem_vec_ty, - llvm_pointer_vec_ty, - alignment_ty, - mask_ty], ret_t)); + let llvm_intrinsic = + format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let f = bx.declare_cfn( + &llvm_intrinsic, + bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t), + ); llvm::SetUnnamedAddr(f, false); - let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], - None); + let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); return Ok(v); } macro_rules! arith_red { ($name:tt : $integer_reduce:ident, $float_reduce:ident, $ordered:expr) => { if name == $name { - require!(ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, in_ty, ret_ty); + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); return match in_elem.kind { ty::Int(_) | ty::Uint(_) => { let r = bx.$integer_reduce(args[0].immediate()); @@ -1659,7 +1794,7 @@ fn generic_simd_intrinsic( } else { Ok(bx.$integer_reduce(args[0].immediate())) } - }, + } ty::Float(f) => { let acc = if $ordered { // ordered arithmetic reductions take an accumulator @@ -1670,25 +1805,29 @@ fn generic_simd_intrinsic( match f.bit_width() { 32 => bx.const_real(bx.type_f32(), identity_acc), 64 => bx.const_real(bx.type_f64(), identity_acc), - v => { - return_error!(r#" + v => return_error!( + r#" unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, - $name, in_ty, in_elem, v, ret_ty - ) - } + $name, + in_ty, + in_elem, + v, + ret_ty + ), } }; Ok(bx.$float_reduce(acc, args[0].immediate())) } - _ => { - return_error!( - "unsupported {} from `{}` with element `{}` to `{}`", - $name, in_ty, in_elem, ret_ty - ) - }, - } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + $name, + in_ty, + in_elem, + ret_ty + ), + }; } - } + }; } arith_red!("simd_reduce_add_ordered": vector_reduce_add, vector_reduce_fadd, true); @@ -1699,27 +1838,27 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, macro_rules! minmax_red { ($name:tt: $int_red:ident, $float_red:ident) => { if name == $name { - require!(ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, in_ty, ret_ty); + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); return match in_elem.kind { - ty::Int(_i) => { - Ok(bx.$int_red(args[0].immediate(), true)) - }, - ty::Uint(_u) => { - Ok(bx.$int_red(args[0].immediate(), false)) - }, - ty::Float(_f) => { - Ok(bx.$float_red(args[0].immediate())) - } - _ => { - return_error!("unsupported {} from `{}` with element `{}` to `{}`", - $name, in_ty, in_elem, ret_ty) - }, - } + ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), + ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), + ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + $name, + in_ty, + in_elem, + ret_ty + ), + }; } - - } + }; } minmax_red!("simd_reduce_min": vector_reduce_min, vector_reduce_fmin); @@ -1732,17 +1871,24 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ($name:tt : $red:ident, $boolean:expr) => { if name == $name { let input = if !$boolean { - require!(ret_ty == in_elem, - "expected return type `{}` (element of input `{}`), found `{}`", - in_elem, in_ty, ret_ty); + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); args[0].immediate() } else { match in_elem.kind { - ty::Int(_) | ty::Uint(_) => {}, - _ => { - return_error!("unsupported {} from `{}` with element `{}` to `{}`", - $name, in_ty, in_elem, ret_ty) - } + ty::Int(_) | ty::Uint(_) => {} + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + $name, + in_ty, + in_elem, + ret_ty + ), } // boolean reductions operate on vectors of i1s: @@ -1753,21 +1899,18 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, return match in_elem.kind { ty::Int(_) | ty::Uint(_) => { let r = bx.$red(input); - Ok( - if !$boolean { - r - } else { - bx.zext(r, bx.type_bool()) - } - ) - }, - _ => { - return_error!("unsupported {} from `{}` with element `{}` to `{}`", - $name, in_ty, in_elem, ret_ty) - }, - } + Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) + } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + $name, + in_ty, + in_elem, + ret_ty + ), + }; } - } + }; } bitwise_red!("simd_reduce_and": vector_reduce_and, false); @@ -1779,17 +1922,27 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, if name == "simd_cast" { require_simd!(ret_ty, "return"); let out_len = ret_ty.simd_size(tcx); - require!(in_len == out_len, - "expected return type with length {} (same as input type `{}`), \ + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ found `{}` with length {}", - in_len, in_ty, - ret_ty, out_len); + in_len, + in_ty, + ret_ty, + out_len + ); // casting cares about nominal type, not just structural type let out_elem = ret_ty.simd_type(tcx); - if in_elem == out_elem { return Ok(args[0].immediate()); } + if in_elem == out_elem { + return Ok(args[0].immediate()); + } - enum Style { Float, Int(/* is signed? */ bool), Unsupported } + enum Style { + Float, + Int(/* is signed? */ bool), + Unsupported, + } let (in_style, in_width) = match in_elem.kind { // vectors of pointer-sized integers should've been @@ -1797,13 +1950,13 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), ty::Float(f) => (Style::Float, f.bit_width()), - _ => (Style::Unsupported, 0) + _ => (Style::Unsupported, 0), }; let (out_style, out_width) = match out_elem.kind { ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), ty::Float(f) => (Style::Float, f.bit_width()), - _ => (Style::Unsupported, 0) + _ => (Style::Unsupported, 0), }; match (in_style, out_style) { @@ -1811,40 +1964,46 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, return Ok(match in_width.cmp(&out_width) { Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), Ordering::Equal => args[0].immediate(), - Ordering::Less => if in_is_signed { - bx.sext(args[0].immediate(), llret_ty) - } else { - bx.zext(args[0].immediate(), llret_ty) + Ordering::Less => { + if in_is_signed { + bx.sext(args[0].immediate(), llret_ty) + } else { + bx.zext(args[0].immediate(), llret_ty) + } } - }) + }); } (Style::Int(in_is_signed), Style::Float) => { return Ok(if in_is_signed { bx.sitofp(args[0].immediate(), llret_ty) } else { bx.uitofp(args[0].immediate(), llret_ty) - }) + }); } (Style::Float, Style::Int(out_is_signed)) => { return Ok(if out_is_signed { bx.fptosi(args[0].immediate(), llret_ty) } else { bx.fptoui(args[0].immediate(), llret_ty) - }) + }); } (Style::Float, Style::Float) => { return Ok(match in_width.cmp(&out_width) { Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), Ordering::Equal => args[0].immediate(), - Ordering::Less => bx.fpext(args[0].immediate(), llret_ty) - }) + Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), + }); } - _ => {/* Unsupported. Fallthrough. */} + _ => { /* Unsupported. Fallthrough. */ } } - require!(false, - "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", - in_ty, in_elem, - ret_ty, out_elem); + require!( + false, + "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", + in_ty, + in_elem, + ret_ty, + out_elem + ); } macro_rules! arith { ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { @@ -1884,23 +2043,14 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, let is_add = name == "simd_saturating_add"; let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; let (signed, elem_width, elem_ty) = match in_elem.kind { - ty::Int(i) => - ( - true, - i.bit_width().unwrap_or(ptr_bits), - bx.cx.type_int_from_ty(i) - ), - ty::Uint(i) => - ( - false, - i.bit_width().unwrap_or(ptr_bits), - bx.cx.type_uint_from_ty(i) - ), + ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), + ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), _ => { return_error!( "expected element type `{}` of vector type `{}` \ to be a signed or unsigned integer type", - arg_tys[0].simd_type(tcx), arg_tys[0] + arg_tys[0].simd_type(tcx), + arg_tys[0] ); } }; @@ -1908,14 +2058,12 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, "llvm.{}{}.sat.v{}i{}", if signed { 's' } else { 'u' }, if is_add { "add" } else { "sub" }, - in_len, elem_width + in_len, + elem_width ); let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - let f = bx.declare_cfn( - &llvm_intrinsic, - bx.type_func(&[vec_ty, vec_ty], vec_ty) - ); + let f = bx.declare_cfn(&llvm_intrinsic, bx.type_func(&[vec_ty, vec_ty], vec_ty)); llvm::SetUnnamedAddr(f, false); let v = bx.call(f, &[lhs, rhs], None); return Ok(v); @@ -1930,22 +2078,28 @@ unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, // stuffs. fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { match ty.kind { - ty::Int(t) => Some((match t { - ast::IntTy::Isize => cx.tcx.sess.target.ptr_width as u64, - ast::IntTy::I8 => 8, - ast::IntTy::I16 => 16, - ast::IntTy::I32 => 32, - ast::IntTy::I64 => 64, - ast::IntTy::I128 => 128, - }, true)), - ty::Uint(t) => Some((match t { - ast::UintTy::Usize => cx.tcx.sess.target.ptr_width as u64, - ast::UintTy::U8 => 8, - ast::UintTy::U16 => 16, - ast::UintTy::U32 => 32, - ast::UintTy::U64 => 64, - ast::UintTy::U128 => 128, - }, false)), + ty::Int(t) => Some(( + match t { + ast::IntTy::Isize => cx.tcx.sess.target.ptr_width as u64, + ast::IntTy::I8 => 8, + ast::IntTy::I16 => 16, + ast::IntTy::I32 => 32, + ast::IntTy::I64 => 64, + ast::IntTy::I128 => 128, + }, + true, + )), + ty::Uint(t) => Some(( + match t { + ast::UintTy::Usize => cx.tcx.sess.target.ptr_width as u64, + ast::UintTy::U8 => 8, + ast::UintTy::U16 => 16, + ast::UintTy::U32 => 32, + ast::UintTy::U64 => 64, + ast::UintTy::U128 => 128, + }, + false, + )), _ => None, } } |