diff options
Diffstat (limited to 'compiler')
| -rw-r--r-- | compiler/rustc_codegen_ssa/Cargo.toml | 4 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/operand.rs | 30 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 81 |
3 files changed, 102 insertions, 13 deletions
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 1562a09f5c1..3771fc6b0a2 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -6,8 +6,10 @@ edition = "2021" [dependencies] # tidy-alphabetical-start ar_archive_writer = "0.2.0" +arrayvec = { version = "0.7", default-features = false } bitflags = "2.4.1" -cc = "1.0.97" +cc = "1.0.90" +either = "1.5.0" itertools = "0.12" jobserver = "0.1.28" pathdiff = "0.2.0" diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 38f77f2e646..4cff6ebd35d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -14,6 +14,9 @@ use rustc_target::abi::{self, Abi, Align, Size}; use std::fmt; +use arrayvec::ArrayVec; +use either::Either; + /// The representation of a Rust value. The enum variant is in fact /// uniquely determined by the value's type, but is kept as a /// safety check. @@ -58,6 +61,33 @@ pub enum OperandValue<V> { ZeroSized, } +impl<V> OperandValue<V> { + /// If this is ZeroSized/Immediate/Pair, return an array of the 0/1/2 values. + /// If this is Ref, return the place. + #[inline] + pub fn immediates_or_place(self) -> Either<ArrayVec<V, 2>, PlaceValue<V>> { + match self { + OperandValue::ZeroSized => Either::Left(ArrayVec::new()), + OperandValue::Immediate(a) => Either::Left(ArrayVec::from_iter([a])), + OperandValue::Pair(a, b) => Either::Left([a, b].into()), + OperandValue::Ref(p) => Either::Right(p), + } + } + + /// Given an array of 0/1/2 immediate values, return ZeroSized/Immediate/Pair. + #[inline] + pub fn from_immediates(immediates: ArrayVec<V, 2>) -> Self { + let mut it = immediates.into_iter(); + let Some(a) = it.next() else { + return OperandValue::ZeroSized; + }; + let Some(b) = it.next() else { + return OperandValue::Immediate(a); + }; + OperandValue::Pair(a, b) + } +} + /// An `OperandRef` is an "SSA" reference to a Rust value, along with /// its type. /// diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 6be473ffa1f..06663a0f61c 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -8,14 +8,16 @@ use crate::traits::*; use crate::MemFlags; use rustc_hir as hir; -use rustc_middle::mir::{self, AggregateKind, Operand}; +use rustc_middle::mir; use rustc_middle::ty::cast::{CastTy, IntTy}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::{Span, DUMMY_SP}; -use rustc_target::abi::{self, FIRST_VARIANT}; +use rustc_target::abi::{self, FieldIdx, FIRST_VARIANT}; + +use arrayvec::ArrayVec; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] @@ -581,7 +583,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_place_to_pointer(bx, place, mk_ref) } - mir::Rvalue::CopyForDeref(place) => self.codegen_operand(bx, &Operand::Copy(place)), + mir::Rvalue::CopyForDeref(place) => { + self.codegen_operand(bx, &mir::Operand::Copy(place)) + } mir::Rvalue::AddressOf(mutability, place) => { let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability); @@ -739,11 +743,40 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"), - mir::Rvalue::Aggregate(..) => { - // According to `rvalue_creates_operand`, only ZST - // aggregate rvalues are allowed to be operands. + mir::Rvalue::Aggregate(_, ref fields) => { let ty = rvalue.ty(self.mir, self.cx.tcx()); - OperandRef::zero_sized(self.cx.layout_of(self.monomorphize(ty))) + let ty = self.monomorphize(ty); + let layout = self.cx.layout_of(ty); + + // `rvalue_creates_operand` has arranged that we only get here if + // we can build the aggregate immediate from the field immediates. + let mut inputs = ArrayVec::<Bx::Value, 2>::new(); + let mut input_scalars = ArrayVec::<abi::Scalar, 2>::new(); + for field_idx in layout.fields.index_by_increasing_offset() { + let field_idx = FieldIdx::from_usize(field_idx); + let op = self.codegen_operand(bx, &fields[field_idx]); + let values = op.val.immediates_or_place().left_or_else(|p| { + bug!("Field {field_idx:?} is {p:?} making {layout:?}"); + }); + inputs.extend(values); + let scalars = self.value_kind(op.layout).scalars().unwrap(); + input_scalars.extend(scalars); + } + + let output_scalars = self.value_kind(layout).scalars().unwrap(); + itertools::izip!(&mut inputs, input_scalars, output_scalars).for_each( + |(v, in_s, out_s)| { + if in_s != out_s { + // We have to be really careful about bool here, because + // `(bool,)` stays i1 but `Cell<bool>` becomes i8. + *v = bx.from_immediate(*v); + *v = bx.to_immediate_scalar(*v, out_s); + } + }, + ); + + let val = OperandValue::from_immediates(inputs); + OperandRef { val, layout } } mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { let operand = self.codegen_operand(bx, operand); @@ -1051,16 +1084,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::ThreadLocalRef(_) | mir::Rvalue::Use(..) => // (*) true, - // This always produces a `ty::RawPtr`, so will be Immediate or Pair - mir::Rvalue::Aggregate(box AggregateKind::RawPtr(..), ..) => true, // Arrays are always aggregates, so it's not worth checking anything here. // (If it's really `[(); N]` or `[T; 0]` and we use the place path, fine.) mir::Rvalue::Repeat(..) => false, - mir::Rvalue::Aggregate(..) => { + mir::Rvalue::Aggregate(ref kind, _) => { + let allowed_kind = match **kind { + // This always produces a `ty::RawPtr`, so will be Immediate or Pair + mir::AggregateKind::RawPtr(..) => true, + mir::AggregateKind::Array(..) => false, + mir::AggregateKind::Tuple => true, + mir::AggregateKind::Adt(def_id, ..) => { + let adt_def = self.cx.tcx().adt_def(def_id); + adt_def.is_struct() && !adt_def.repr().simd() + } + mir::AggregateKind::Closure(..) => true, + // FIXME: Can we do this for simple coroutines too? + mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false, + }; + allowed_kind && { let ty = rvalue.ty(self.mir, self.cx.tcx()); let ty = self.monomorphize(ty); - // For ZST this can be `OperandValueKind::ZeroSized`. - self.cx.spanned_layout_of(ty, span).is_zst() + let layout = self.cx.spanned_layout_of(ty, span); + !self.cx.is_backend_ref(layout) + } } } @@ -1102,3 +1148,14 @@ enum OperandValueKind { Pair(abi::Scalar, abi::Scalar), ZeroSized, } + +impl OperandValueKind { + fn scalars(self) -> Option<ArrayVec<abi::Scalar, 2>> { + Some(match self { + OperandValueKind::ZeroSized => ArrayVec::new(), + OperandValueKind::Immediate(a) => ArrayVec::from_iter([a]), + OperandValueKind::Pair(a, b) => [a, b].into(), + OperandValueKind::Ref => return None, + }) + } +} |