diff options
Diffstat (limited to 'compiler/rustc_codegen_gcc')
| -rw-r--r-- | compiler/rustc_codegen_gcc/example/alloc_system.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch | 1 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/abi.rs | 39 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/builder.rs | 174 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/common.rs | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/consts.rs | 12 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 20 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/src/lib.rs | 1 |
8 files changed, 205 insertions, 48 deletions
diff --git a/compiler/rustc_codegen_gcc/example/alloc_system.rs b/compiler/rustc_codegen_gcc/example/alloc_system.rs index 5f66ca67f2d..89661918d05 100644 --- a/compiler/rustc_codegen_gcc/example/alloc_system.rs +++ b/compiler/rustc_codegen_gcc/example/alloc_system.rs @@ -156,7 +156,7 @@ mod platform { struct Header(*mut u8); const HEAP_ZERO_MEMORY: DWORD = 0x00000008; unsafe fn get_header<'a>(ptr: *mut u8) -> &'a mut Header { - &mut *(ptr as *mut Header).offset(-1) + &mut *(ptr as *mut Header).sub(1) } unsafe fn align_ptr(ptr: *mut u8, align: usize) -> *mut u8 { let aligned = ptr.add(align - (ptr as usize & (align - 1))); diff --git a/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch b/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch index d5fa1cec061..c59a40df039 100644 --- a/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch +++ b/compiler/rustc_codegen_gcc/patches/0024-core-Disable-portable-simd-test.patch @@ -14,7 +14,6 @@ index 06c7be0..359e2e7 100644 @@ -75,7 +75,6 @@ #![feature(never_type)] #![feature(unwrap_infallible)] - #![feature(result_into_ok_or_err)] -#![feature(portable_simd)] #![feature(ptr_metadata)] #![feature(once_cell)] diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs index 0ed3e1fbe93..848c34211ff 100644 --- a/compiler/rustc_codegen_gcc/src/abi.rs +++ b/compiler/rustc_codegen_gcc/src/abi.rs @@ -107,45 +107,24 @@ pub trait FnAbiGccExt<'gcc, 'tcx> { impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>) { let mut on_stack_param_indices = FxHashSet::default(); - let args_capacity: usize = self.args.iter().map(|arg| - if arg.pad.is_some() { - 1 - } - else { - 0 - } + - if let PassMode::Pair(_, _) = arg.mode { - 2 - } else { - 1 - } - ).sum(); + + // This capacity calculation is approximate. let mut argument_tys = Vec::with_capacity( - if let PassMode::Indirect { .. } = self.ret.mode { - 1 - } - else { - 0 - } + args_capacity, + self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } ); let return_ty = match self.ret.mode { PassMode::Ignore => cx.type_void(), PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_gcc_type(cx), - PassMode::Cast(cast) => cast.gcc_type(cx), + PassMode::Cast(ref cast, _) => cast.gcc_type(cx), PassMode::Indirect { .. } => { argument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); cx.type_void() } }; - for arg in &self.args { - // add padding - if let Some(ty) = arg.pad { - argument_tys.push(ty.gcc_type(cx)); - } - + for arg in self.args.iter() { let arg_ty = match arg.mode { PassMode::Ignore => continue, PassMode::Direct(_) => arg.layout.immediate_gcc_type(cx), @@ -157,7 +136,13 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { extra_attrs: Some(_), .. } => { unimplemented!(); } - PassMode::Cast(cast) => cast.gcc_type(cx), + PassMode::Cast(ref cast, pad_i32) => { + // add padding + if pad_i32 { + argument_tys.push(Reg::i32().gcc_type(cx)); + } + cast.gcc_type(cx) + } PassMode::Indirect { extra_attrs: None, on_stack: true, .. } => { on_stack_param_indices.insert(argument_tys.len()); arg.memory_ty(cx) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 4d40dd0994d..6994eeb00c3 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -15,8 +15,11 @@ use gccjit::{ Type, UnaryOp, }; +use rustc_apfloat::{ieee, Float, Round, Status}; use rustc_codegen_ssa::MemFlags; -use rustc_codegen_ssa::common::{AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope}; +use rustc_codegen_ssa::common::{ + AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind, +}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{ @@ -31,6 +34,7 @@ use rustc_codegen_ssa::traits::{ StaticBuilderMethods, }; use rustc_data_structures::fx::FxHashSet; +use rustc_middle::bug; use rustc_middle::ty::{ParamEnv, Ty, TyCtxt}; use rustc_middle::ty::layout::{FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout}; use rustc_span::Span; @@ -1271,12 +1275,12 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { val } - fn fptoui_sat(&mut self, _val: RValue<'gcc>, _dest_ty: Type<'gcc>) -> Option<RValue<'gcc>> { - None + fn fptoui_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> { + self.fptoint_sat(false, val, dest_ty) } - fn fptosi_sat(&mut self, _val: RValue<'gcc>, _dest_ty: Type<'gcc>) -> Option<RValue<'gcc>> { - None + fn fptosi_sat(&mut self, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> { + self.fptoint_sat(true, val, dest_ty) } fn instrprof_increment(&mut self, _fn_name: RValue<'gcc>, _hash: RValue<'gcc>, _num_counters: RValue<'gcc>, _index: RValue<'gcc>) { @@ -1285,6 +1289,166 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { } impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { + fn fptoint_sat(&mut self, signed: bool, val: RValue<'gcc>, dest_ty: Type<'gcc>) -> RValue<'gcc> { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty) = if self.cx.type_kind(src_ty) == TypeKind::Vector { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + (self.cx.element_type(src_ty), self.cx.element_type(dest_ty)) + } else { + (src_ty, dest_ty) + }; + + // FIXME(jistone): the following was originally the fallback SSA implementation, before LLVM 13 + // added native `fptosi.sat` and `fptoui.sat` conversions, but it was used by GCC as well. + // Now that LLVM always relies on its own, the code has been moved to GCC, but the comments are + // still LLVM-specific. This should be updated, and use better GCC specifics if possible. + + let int_width = self.cx.int_width(int_ty); + let float_width = self.cx.float_width(float_ty); + // LLVM's fpto[su]i returns undef when the input val is infinite, NaN, or does not fit into the + // destination integer type after rounding towards zero. This `undef` value can cause UB in + // safe code (see issue #10184), so we implement a saturating conversion on top of it: + // Semantically, the mathematical value of the input is rounded towards zero to the next + // mathematical integer, and then the result is clamped into the range of the destination + // integer type. Positive and negative infinity are mapped to the maximum and minimum value of + // the destination integer type. NaN is mapped to 0. + // + // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to + // a value representable in int_ty. + // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits. + // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two. + // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly + // representable. Note that this only works if float_ty's exponent range is sufficiently large. + // f16 or 256 bit integers would break this property. Right now the smallest float type is f32 + // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127. + // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because + // we're rounding towards zero, we just get float_ty::MAX (which is always an integer). + // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX. + let int_max = |signed: bool, int_width: u64| -> u128 { + let shift_amount = 128 - int_width; + if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount } + }; + let int_min = |signed: bool, int_width: u64| -> i128 { + if signed { i128::MIN >> (128 - int_width) } else { 0 } + }; + + let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) { + let rounded_min = + ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero); + assert_eq!(rounded_min.status, Status::OK); + let rounded_max = + ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero); + assert!(rounded_max.value.is_finite()); + (rounded_min.value.to_bits(), rounded_max.value.to_bits()) + }; + // To implement saturation, we perform the following steps: + // + // 1. Cast val to an integer with fpto[su]i. This may result in undef. + // 2. Compare val to f_min and f_max, and use the comparison results to select: + // a) int_ty::MIN if val < f_min or val is NaN + // b) int_ty::MAX if val > f_max + // c) the result of fpto[su]i otherwise + // 3. If val is NaN, return 0.0, otherwise return the result of step 2. + // + // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the + // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of + // undef does not introduce any non-determinism either. + // More importantly, the above procedure correctly implements saturating conversion. + // Proof (sketch): + // If val is NaN, 0 is returned by definition. + // Otherwise, val is finite or infinite and thus can be compared with f_min and f_max. + // This yields three cases to consider: + // (1) if val in [f_min, f_max], the result of fpto[su]i is returned, which agrees with + // saturating conversion for inputs in that range. + // (2) if val > f_max, then val is larger than int_ty::MAX. This holds even if f_max is rounded + // (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger + // than int_ty::MAX. Because val is larger than int_ty::MAX, the return value of int_ty::MAX + // is correct. + // (3) if val < f_min, then val is smaller than int_ty::MIN. As shown earlier, f_min exactly equals + // int_ty::MIN and therefore the return value of int_ty::MIN is correct. + // QED. + + let float_bits_to_llval = |bx: &mut Self, bits| { + let bits_llval = match float_width { + 32 => bx.cx().const_u32(bits as u32), + 64 => bx.cx().const_u64(bits as u64), + n => bug!("unsupported float width {}", n), + }; + bx.bitcast(bits_llval, float_ty) + }; + let (f_min, f_max) = match float_width { + 32 => compute_clamp_bounds_single(signed, int_width), + 64 => compute_clamp_bounds_double(signed, int_width), + n => bug!("unsupported float width {}", n), + }; + let f_min = float_bits_to_llval(self, f_min); + let f_max = float_bits_to_llval(self, f_max); + let int_max = self.cx.const_uint_big(int_ty, int_max(signed, int_width)); + let int_min = self.cx.const_uint_big(int_ty, int_min(signed, int_width) as u128); + let zero = self.cx.const_uint(int_ty, 0); + + // If we're working with vectors, constants must be "splatted": the constant is duplicated + // into each lane of the vector. The algorithm stays the same, we are just using the + // same constant across all lanes. + let maybe_splat = |bx: &mut Self, val| { + if bx.cx().type_kind(dest_ty) == TypeKind::Vector { + bx.vector_splat(bx.vector_length(dest_ty), val) + } else { + val + } + }; + let f_min = maybe_splat(self, f_min); + let f_max = maybe_splat(self, f_max); + let int_max = maybe_splat(self, int_max); + let int_min = maybe_splat(self, int_min); + let zero = maybe_splat(self, zero); + + // Step 1 ... + let fptosui_result = if signed { self.fptosi(val, dest_ty) } else { self.fptoui(val, dest_ty) }; + let less_or_nan = self.fcmp(RealPredicate::RealULT, val, f_min); + let greater = self.fcmp(RealPredicate::RealOGT, val, f_max); + + // Step 2: We use two comparisons and two selects, with %s1 being the + // result: + // %less_or_nan = fcmp ult %val, %f_min + // %greater = fcmp olt %val, %f_max + // %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result + // %s1 = select %greater, int_ty::MAX, %s0 + // Note that %less_or_nan uses an *unordered* comparison. This + // comparison is true if the operands are not comparable (i.e., if val is + // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if + // val is NaN. + // + // Performance note: Unordered comparison can be lowered to a "flipped" + // comparison and a negation, and the negation can be merged into the + // select. Therefore, it not necessarily any more expensive than an + // ordered ("normal") comparison. Whether these optimizations will be + // performed is ultimately up to the backend, but at least x86 does + // perform them. + let s0 = self.select(less_or_nan, int_min, fptosui_result); + let s1 = self.select(greater, int_max, s0); + + // Step 3: NaN replacement. + // For unsigned types, the above step already yielded int_ty::MIN == 0 if val is NaN. + // Therefore we only need to execute this step for signed integer types. + if signed { + // LLVM has no isNaN predicate, so we use (val == val) instead + let cmp = self.fcmp(RealPredicate::RealOEQ, val, val); + self.select(cmp, s1, zero) + } else { + s1 + } + } + #[cfg(feature="master")] pub fn shuffle_vector(&mut self, v1: RValue<'gcc>, v2: RValue<'gcc>, mask: RValue<'gcc>) -> RValue<'gcc> { let struct_type = mask.get_type().is_struct().expect("mask of struct type"); diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs index ccb6cbbc2c8..aa1c271c31c 100644 --- a/compiler/rustc_codegen_gcc/src/common.rs +++ b/compiler/rustc_codegen_gcc/src/common.rs @@ -158,10 +158,6 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> { None } - fn zst_to_backend(&self, _ty: Type<'gcc>) -> RValue<'gcc> { - self.const_undef(self.type_ix(0)) - } - fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, ty: Type<'gcc>) -> RValue<'gcc> { let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; match cv { diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs index c0b8d21818f..356c03ee3c1 100644 --- a/compiler/rustc_codegen_gcc/src/consts.rs +++ b/compiler/rustc_codegen_gcc/src/consts.rs @@ -127,7 +127,7 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> { // // We could remove this hack whenever we decide to drop macOS 10.10 support. if self.tcx.sess.target.options.is_like_osx { - // The `inspect` method is okay here because we checked relocations, and + // The `inspect` method is okay here because we checked for provenance, and // because we are doing this access to inspect the final interpreter state // (not as part of the interpreter execution). // @@ -296,17 +296,17 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAllocation<'tcx>) -> RValue<'gcc> { let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let mut llvals = Vec::with_capacity(alloc.provenance().len() + 1); let dl = cx.data_layout(); let pointer_size = dl.pointer_size.bytes() as usize; let mut next_offset = 0; - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().iter() { let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a relocation, it + // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). Any undef byte is replaced with // some arbitrary byte value. @@ -319,7 +319,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAl read_target_uint( dl.endian, // This `inspect` is okay since it is within the bounds of the allocation, it doesn't // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the relocation as a relocation pointer offset. + // and we properly interpret the provenance as a relocation pointer offset. alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), ) .expect("const_alloc_to_llvm: could not read relocation pointer") @@ -336,7 +336,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAl } if alloc.len() >= next_offset { let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all relocations, it is + // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). Any undef byte is replaced with some // arbitrary byte value. diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 5fbdedac0c4..02cedd4646b 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -130,7 +130,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { sym::volatile_load | sym::unaligned_volatile_load => { let tp_ty = substs.type_at(0); let mut ptr = args[0].immediate(); - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { ptr = self.pointercast(ptr, self.type_ptr_to(ty.gcc_type(self))); } let load = self.volatile_load(ptr.get_type(), ptr); @@ -309,6 +309,18 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { return; } + sym::ptr_mask => { + let usize_type = self.context.new_type::<usize>(); + let void_ptr_type = self.context.new_type::<*const ()>(); + + let ptr = args[0].immediate(); + let mask = args[1].immediate(); + + let addr = self.bitcast(ptr, usize_type); + let masked = self.and(addr, mask); + self.bitcast(masked, void_ptr_type) + }, + _ if name_str.starts_with("simd_") => { match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, @@ -320,7 +332,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { }; if !fn_abi.ret.is_ignore() { - if let PassMode::Cast(ty) = fn_abi.ret.mode { + if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = self.type_ptr_to(ty.gcc_type(self)); let ptr = self.pointercast(result.llval, ptr_llty); self.store(llval, ptr, result.align); @@ -416,7 +428,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { else if self.is_unsized_indirect() { bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); } - else if let PassMode::Cast(cast) = self.mode { + else if let PassMode::Cast(ref cast, _) = self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. let can_store_through_cast_ptr = false; @@ -481,7 +493,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { PassMode::Indirect { extra_attrs: Some(_), .. } => { OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); }, - PassMode::Direct(_) | PassMode::Indirect { extra_attrs: None, .. } | PassMode::Cast(_) => { + PassMode::Direct(_) | PassMode::Indirect { extra_attrs: None, .. } | PassMode::Cast(..) => { let next_arg = next(); self.store(bx, next_arg, dst); }, diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 8a206c0368f..223466fb9b5 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -19,6 +19,7 @@ #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] +extern crate rustc_apfloat; extern crate rustc_ast; extern crate rustc_codegen_ssa; extern crate rustc_data_structures; |