Merge from rustc

9 files changed, 67 insertions, 19 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index bea12747a51..70b45a852ca 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -1,4 +1,5 @@
 use std::any::Any;
+use std::assert_matches::assert_matches;
 use std::marker::PhantomData;
 use std::path::{Path, PathBuf};
 use std::sync::mpsc::{channel, Receiver, Sender};
@@ -1963,7 +1964,7 @@ impl SharedEmitterMain {
                     sess.dcx().abort_if_errors();
                 }
                 Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => {
-                    assert!(matches!(level, Level::Error | Level::Warning | Level::Note));
+                    assert_matches!(level, Level::Error | Level::Warning | Level::Note);
                     let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
                     let mut err = Diag::<()>::new(sess.dcx(), level, msg);
 
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 27558038927..359043a6d78 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -331,7 +331,7 @@ fn push_debuginfo_type_name<'tcx>(
                 output.push(')');
             }
         }
-        ty::FnDef(..) | ty::FnPtr(_) => {
+        ty::FnDef(..) | ty::FnPtr(..) => {
             // We've encountered a weird 'recursive type'
             // Currently, the only way to generate such a type
             // is by using 'impl trait':
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 1b029660433..cb6d9d6f66e 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -4,6 +4,7 @@
 #![allow(rustc::untranslatable_diagnostic)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
+#![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index bc3076528da..817e2ca72ec 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -846,7 +846,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 ),
                 None,
             ),
-            ty::FnPtr(_) => (None, Some(callee.immediate())),
+            ty::FnPtr(..) => (None, Some(callee.immediate())),
             _ => bug!("{} is not callable", callee.layout.ty),
         };
 
@@ -923,8 +923,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         // third argument must be constant. This is
                         // checked by the type-checker.
                         if i == 2 && intrinsic.name == sym::simd_shuffle {
+                            // FIXME: the simd_shuffle argument is actually an array,
+                            // not a vector, so we need this special hack to make sure
+                            // it is passed as an immediate. We should pass the
+                            // shuffle indices as a vector instead to avoid this hack.
                             if let mir::Operand::Constant(constant) = &arg.node {
-                                let (llval, ty) = self.simd_shuffle_indices(bx, constant);
+                                let (llval, ty) = self.immediate_const_vector(bx, constant);
                                 return OperandRef {
                                     val: Immediate(llval),
                                     layout: bx.layout_of(ty),
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index f4d974f7036..0aa85b82038 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -1,6 +1,6 @@
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::layout::HasTyCtxt;
-use rustc_middle::ty::{self, Ty};
+use rustc_middle::ty::{self, Ty, ValTree};
 use rustc_middle::{bug, mir, span_bug};
 use rustc_target::abi::Abi;
 
@@ -28,7 +28,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             .expect("erroneous constant missed by mono item collection")
     }
 
-    /// This is a convenience helper for `simd_shuffle_indices`. It has the precondition
+    /// This is a convenience helper for `immediate_const_vector`. It has the precondition
     /// that the given `constant` is an `Const::Unevaluated` and must be convertible to
     /// a `ValTree`. If you want a more general version of this, talk to `wg-const-eval` on zulip.
     ///
@@ -59,23 +59,42 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         self.cx.tcx().const_eval_resolve_for_typeck(ty::ParamEnv::reveal_all(), uv, constant.span)
     }
 
-    /// process constant containing SIMD shuffle indices
-    pub fn simd_shuffle_indices(
+    /// process constant containing SIMD shuffle indices & constant vectors
+    pub fn immediate_const_vector(
         &mut self,
         bx: &Bx,
         constant: &mir::ConstOperand<'tcx>,
     ) -> (Bx::Value, Ty<'tcx>) {
         let ty = self.monomorphize(constant.ty());
+        let ty_is_simd = ty.is_simd();
+        // FIXME: ideally we'd assert that this is a SIMD type, but simd_shuffle
+        // in its current form relies on a regular array being passed as an
+        // immediate argument. This hack can be removed once that is fixed.
+        let field_ty = if ty_is_simd {
+            ty.simd_size_and_type(bx.tcx()).1
+        } else {
+            ty.builtin_index().unwrap()
+        };
+
         let val = self
             .eval_unevaluated_mir_constant_to_valtree(constant)
             .ok()
             .map(|x| x.ok())
             .flatten()
             .map(|val| {
-                let field_ty = ty.builtin_index().unwrap();
-                let values: Vec<_> = val
-                    .unwrap_branch()
-                    .iter()
+                // Depending on whether this is a SIMD type with an array field
+                // or a type with many fields (one for each elements), the valtree
+                // is either a single branch with N children, or a root node
+                // with exactly one child which then in turn has many children.
+                // So we look at the first child to determine whether it is a
+                // leaf or whether we have to go one more layer down.
+                let branch_or_leaf = val.unwrap_branch();
+                let first = branch_or_leaf.get(0).unwrap();
+                let field_iter = match first {
+                    ValTree::Branch(_) => first.unwrap_branch().iter(),
+                    ValTree::Leaf(_) => branch_or_leaf.iter(),
+                };
+                let values: Vec<_> = field_iter
                     .map(|field| {
                         if let Some(prim) = field.try_to_scalar() {
                             let layout = bx.layout_of(field_ty);
@@ -84,11 +103,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             };
                             bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout))
                         } else {
-                            bug!("simd shuffle field {:?}", field)
+                            bug!("field is not a scalar {:?}", field)
                         }
                     })
                     .collect();
-                bx.const_struct(&values, false)
+                if ty_is_simd { bx.const_vector(&values) } else { bx.const_struct(&values, false) }
             })
             .unwrap_or_else(|| {
                 bx.tcx().dcx().emit_err(errors::ShuffleIndicesEvaluation { span: constant.span });
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 2bc2d0f70bf..1891de8c0eb 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -1,3 +1,4 @@
+use std::assert_matches::assert_matches;
 use std::fmt;
 
 use arrayvec::ArrayVec;
@@ -389,7 +390,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             }
             // Newtype vector of array, e.g. #[repr(simd)] struct S([i32; 4]);
             (OperandValue::Immediate(llval), Abi::Aggregate { sized: true }) => {
-                assert!(matches!(self.layout.abi, Abi::Vector { .. }));
+                assert_matches!(self.layout.abi, Abi::Vector { .. });
 
                 let llfield_ty = bx.cx().backend_type(field);
 
@@ -635,7 +636,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_consume(bx, place.as_ref())
             }
 
-            mir::Operand::Constant(ref constant) => self.eval_mir_constant_to_operand(bx, constant),
+            mir::Operand::Constant(ref constant) => {
+                let constant_ty = self.monomorphize(constant.ty());
+                // Most SIMD vector constants should be passed as immediates.
+                // (In particular, some intrinsics really rely on this.)
+                if constant_ty.is_simd() {
+                    // However, some SIMD types do not actually use the vector ABI
+                    // (in particular, packed SIMD types do not). Ensure we exclude those.
+                    let layout = bx.layout_of(constant_ty);
+                    if let Abi::Vector { .. } = layout.abi {
+                        let (llval, ty) = self.immediate_const_vector(bx, constant);
+                        return OperandRef {
+                            val: OperandValue::Immediate(llval),
+                            layout: bx.layout_of(ty),
+                        };
+                    }
+                }
+                self.eval_mir_constant_to_operand(bx, constant)
+            }
         }
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index d91a118bc71..3c2c29ac7f7 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -1,3 +1,5 @@
+use std::assert_matches::assert_matches;
+
 use arrayvec::ArrayVec;
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
@@ -220,7 +222,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         match operand.val {
             OperandValue::Ref(source_place_val) => {
                 assert_eq!(source_place_val.llextra, None);
-                assert!(matches!(operand_kind, OperandValueKind::Ref));
+                assert_matches!(operand_kind, OperandValueKind::Ref);
                 Some(bx.load_operand(source_place_val.with_type(cast)).val)
             }
             OperandValue::ZeroSized => {
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 2b802240e03..6cf84a012f0 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -1,3 +1,5 @@
+use std::assert_matches::assert_matches;
+
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout};
 use rustc_middle::ty::{Instance, Ty};
@@ -254,10 +256,10 @@ pub trait BuilderMethods<'a, 'tcx>:
         } else {
             (in_ty, dest_ty)
         };
-        assert!(matches!(
+        assert_matches!(
             self.cx().type_kind(float_ty),
             TypeKind::Half | TypeKind::Float | TypeKind::Double | TypeKind::FP128
-        ));
+        );
         assert_eq!(self.cx().type_kind(int_ty), TypeKind::Integer);
 
         if let Some(false) = self.cx().sess().opts.unstable_opts.saturating_float_casts {
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index d93b3e06ca6..c15f1fa8e56 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -30,6 +30,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
 
     fn const_str(&self, s: &str) -> (Self::Value, Self::Value);
     fn const_struct(&self, elts: &[Self::Value], packed: bool) -> Self::Value;
+    fn const_vector(&self, elts: &[Self::Value]) -> Self::Value;
 
     fn const_to_opt_uint(&self, v: Self::Value) -> Option<u64>;
     fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option<u128>;