diff options
Diffstat (limited to 'compiler')
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/intrinsic.rs | 24 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/traits/builder.rs | 54 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/traits/type_.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_const_eval/src/interpret/intrinsics.rs | 25 | ||||
| -rw-r--r-- | compiler/rustc_hir_analysis/src/check/intrinsic.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_span/src/symbol.rs | 1 |
6 files changed, 116 insertions, 4 deletions
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 5532ff6e6a5..3e6cf0ece29 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -9,6 +9,7 @@ use crate::traits::*; use crate::MemFlags; use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_session::config::OptLevel; use rustc_span::{sym, Span}; use rustc_target::abi::{ call::{FnAbi, PassMode}, @@ -75,6 +76,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let name = bx.tcx().item_name(def_id); let name_str = name.as_str(); + // If we're swapping something that's *not* an `OperandValue::Ref`, + // then we can do it directly and avoid the alloca. + // Otherwise, we'll let the fallback MIR body take care of it. + if let sym::typed_swap = name { + let pointee_ty = fn_args.type_at(0); + let pointee_layout = bx.layout_of(pointee_ty); + if !bx.is_backend_ref(pointee_layout) + // But if we're not going to optimize, trying to use the fallback + // body just makes things worse, so don't bother. + || bx.sess().opts.optimize == OptLevel::No + // NOTE(eddyb) SPIR-V's Logical addressing model doesn't allow for arbitrary + // reinterpretation of values as (chunkable) byte arrays, and the loop in the + // block optimization in `ptr::swap_nonoverlapping` is hard to rewrite back + // into the (unoptimized) direct swapping implementation, so we disable it. + || bx.sess().target.arch == "spirv" + { + let x_place = PlaceRef::new_sized(args[0].immediate(), pointee_layout); + let y_place = PlaceRef::new_sized(args[1].immediate(), pointee_layout); + bx.typed_place_swap(x_place, y_place); + return Ok(()); + } + } + let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 36f37e3791b..7bc9dee3a89 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -1,22 +1,24 @@ use super::abi::AbiBuilderMethods; use super::asm::AsmBuilderMethods; +use super::consts::ConstMethods; use super::coverageinfo::CoverageInfoBuilderMethods; use super::debuginfo::DebugInfoBuilderMethods; use super::intrinsic::IntrinsicCallMethods; use super::misc::MiscMethods; -use super::type_::{ArgAbiMethods, BaseTypeMethods}; +use super::type_::{ArgAbiMethods, BaseTypeMethods, LayoutTypeMethods}; use super::{HasCodegen, StaticBuilderMethods}; use crate::common::{ AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind, }; -use crate::mir::operand::OperandRef; +use crate::mir::operand::{OperandRef, OperandValue}; use crate::mir::place::PlaceRef; use crate::MemFlags; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout}; use rustc_middle::ty::Ty; +use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Abi, Align, Scalar, Size, WrappingRange}; @@ -267,6 +269,54 @@ pub trait BuilderMethods<'a, 'tcx>: flags: MemFlags, ); + /// *Typed* copy for non-overlapping places. + /// + /// Has a default implementation in terms of `memcpy`, but specific backends + /// can override to do something smarter if possible. + /// + /// (For example, typed load-stores with alias metadata.) + fn typed_place_copy( + &mut self, + dst: PlaceRef<'tcx, Self::Value>, + src: PlaceRef<'tcx, Self::Value>, + ) { + debug_assert!(src.llextra.is_none()); + debug_assert!(dst.llextra.is_none()); + debug_assert_eq!(dst.layout.size, src.layout.size); + if self.sess().opts.optimize == OptLevel::No && self.is_backend_immediate(dst.layout) { + // If we're not optimizing, the aliasing information from `memcpy` + // isn't useful, so just load-store the value for smaller code. + let temp = self.load_operand(src); + temp.val.store(self, dst); + } else if !dst.layout.is_zst() { + let bytes = self.const_usize(dst.layout.size.bytes()); + self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, MemFlags::empty()); + } + } + + /// *Typed* swap for non-overlapping places. + /// + /// Avoids `alloca`s for Immediates and ScalarPairs. + /// + /// FIXME: Maybe do something smarter for Ref types too? + /// For now, the `typed_swap` intrinsic just doesn't call this for those + /// cases (in non-debug), preferring the fallback body instead. + fn typed_place_swap( + &mut self, + left: PlaceRef<'tcx, Self::Value>, + right: PlaceRef<'tcx, Self::Value>, + ) { + let mut temp = self.load_operand(left); + if let OperandValue::Ref(..) = temp.val { + // The SSA value isn't stand-alone, so we need to copy it elsewhere + let alloca = PlaceRef::alloca(self, left.layout); + self.typed_place_copy(alloca, left); + temp = self.load_operand(alloca); + } + self.typed_place_copy(left, right); + temp.val.store(self, right); + } + fn select( &mut self, cond: Self::Value, diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 72cce43a3fa..555833759eb 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -120,6 +120,20 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> { immediate: bool, ) -> Self::Type; + /// A type that produces an [`OperandValue::Ref`] when loaded. + /// + /// AKA one that's not a ZST, not `is_backend_immediate`, and + /// not `is_backend_scalar_pair`. For such a type, a + /// [`load_operand`] doesn't actually `load` anything. + /// + /// [`OperandValue::Ref`]: crate::mir::operand::OperandValue::Ref + /// [`load_operand`]: super::BuilderMethods::load_operand + fn is_backend_ref(&self, layout: TyAndLayout<'tcx>) -> bool { + !(layout.is_zst() + || self.is_backend_immediate(layout) + || self.is_backend_scalar_pair(layout)) + } + /// A type that can be used in a [`super::BuilderMethods::load`] + /// [`super::BuilderMethods::store`] pair to implement a *typed* copy, /// such as a MIR `*_0 = *_1`. diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index a84ef4ce08e..a8478f721c7 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -21,8 +21,8 @@ use rustc_span::symbol::{sym, Symbol}; use rustc_target::abi::Size; use super::{ - util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, MPlaceTy, Machine, OpTy, - Pointer, + memory::MemoryKind, util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, + MPlaceTy, Machine, OpTy, Pointer, }; use crate::fluent_generated as fluent; @@ -414,6 +414,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let result = self.raw_eq_intrinsic(&args[0], &args[1])?; self.write_scalar(result, dest)?; } + sym::typed_swap => { + self.typed_swap_intrinsic(&args[0], &args[1])?; + } sym::vtable_size => { let ptr = self.read_pointer(&args[0])?; @@ -607,6 +610,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.mem_copy(src, dst, size, nonoverlapping) } + /// Does a *typed* swap of `*left` and `*right`. + fn typed_swap_intrinsic( + &mut self, + left: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, + right: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, + ) -> InterpResult<'tcx> { + let left = self.deref_pointer(left)?; + let right = self.deref_pointer(right)?; + debug_assert_eq!(left.layout, right.layout); + let kind = MemoryKind::Stack; + let temp = self.allocate(left.layout, kind)?; + self.copy_op(&left, &temp)?; + self.copy_op(&right, &left)?; + self.copy_op(&temp, &right)?; + self.deallocate_ptr(temp.ptr(), None, kind)?; + Ok(()) + } + pub(crate) fn write_bytes_intrinsic( &mut self, dst: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 0b526a8c977..1d5d4a3205c 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -484,6 +484,8 @@ pub fn check_intrinsic_type( (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx)) } + sym::typed_swap => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)); 2], Ty::new_unit(tcx)), + sym::discriminant_value => { let assoc_items = tcx.associated_item_def_ids( tcx.require_lang_item(hir::LangItem::DiscriminantKind, None), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index b6c07e8737f..cae860cf2f7 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1836,6 +1836,7 @@ symbols! { type_macros, type_name, type_privacy_lints, + typed_swap, u128, u128_legacy_const_max, u128_legacy_const_min, |