Rollup merge of #41206 - eddyb:avoid-illegal-vectors, r=nagisa

Fix pairs of doubles using an illegal <8 x i8> vector.

Accidentally introduced in #40658 and discovered in some Objective-C bindings (returning `NSPoint`).
Turns out LLVM will widen element types of illegal vectors instead of increasing element count, i.e. it will zero-extend `<8 x i8>` to `<8 x i16>`, interleaving the bytes, instead of using the first 8 of `<16 x i8>`.

4 files changed, 40 insertions, 9 deletions

diff --git a/src/librustc_trans/abi.rs b/src/librustc_trans/abi.rs
index 7be80a757ca..c4fdc46d030 100644
--- a/src/librustc_trans/abi.rs
+++ b/src/librustc_trans/abi.rs
@@ -283,7 +283,7 @@ impl<'tcx> LayoutExt<'tcx> for TyLayout<'tcx> {
 
             Layout::Vector { .. } => {
                 Some(Reg {
-                    kind: RegKind::Integer,
+                    kind: RegKind::Vector,
                     size: self.size(ccx)
                 })
             }
diff --git a/src/librustc_trans/cabi_x86_64.rs b/src/librustc_trans/cabi_x86_64.rs
index cbe170d8583..2daebf5cf3d 100644
--- a/src/librustc_trans/cabi_x86_64.rs
+++ b/src/librustc_trans/cabi_x86_64.rs
@@ -173,14 +173,15 @@ fn reg_component(cls: &[Class], i: &mut usize, size: u64) -> Option<Reg> {
         Class::Sse => {
             let vec_len = 1 + cls[*i+1..].iter().take_while(|&&c| c == Class::SseUp).count();
             *i += vec_len;
-            Some(match size {
-                4 => Reg::f32(),
-                8 => Reg::f64(),
-                _ => {
-                    Reg {
-                        kind: RegKind::Vector,
-                        size: Size::from_bytes(vec_len as u64 * 8)
-                    }
+            Some(if vec_len == 1 {
+                match size {
+                    4 => Reg::f32(),
+                    _ => Reg::f64()
+                }
+            } else {
+                Reg {
+                    kind: RegKind::Vector,
+                    size: Size::from_bytes(vec_len as u64 * 8)
                 }
             })
         }
diff --git a/src/test/run-make/extern-fn-struct-passing-abi/test.c b/src/test/run-make/extern-fn-struct-passing-abi/test.c
index 4253767ee76..4e09928edc6 100644
--- a/src/test/run-make/extern-fn-struct-passing-abi/test.c
+++ b/src/test/run-make/extern-fn-struct-passing-abi/test.c
@@ -38,6 +38,11 @@ struct Huge {
     int32_t e;
 };
 
+struct FloatPoint {
+    double x;
+    double y;
+};
+
 // System V x86_64 ABI:
 // a, b, c, d, e should be in registers
 // s should be byval pointer
@@ -258,3 +263,17 @@ struct Huge huge_struct(struct Huge s) {
 
     return s;
 }
+
+// System V x86_64 ABI:
+// p should be in registers
+// return should be in registers
+//
+// Win64 ABI:
+// p should be a byval pointer
+// return should be in a hidden sret pointer
+struct FloatPoint float_point(struct FloatPoint p) {
+    assert(p.x == 5.);
+    assert(p.y == -3.);
+
+    return p;
+}
diff --git a/src/test/run-make/extern-fn-struct-passing-abi/test.rs b/src/test/run-make/extern-fn-struct-passing-abi/test.rs
index b91362b8edc..ff845a644b1 100644
--- a/src/test/run-make/extern-fn-struct-passing-abi/test.rs
+++ b/src/test/run-make/extern-fn-struct-passing-abi/test.rs
@@ -46,6 +46,13 @@ struct Huge {
     e: i32
 }
 
+#[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(C)]
+struct FloatPoint {
+    x: f64,
+    y: f64
+}
+
 #[link(name = "test", kind = "static")]
 extern {
     fn byval_rect(a: i32, b: i32, c: i32, d: i32, e: i32, s: Rect);
@@ -72,6 +79,8 @@ extern {
     fn sret_split_struct(a: i32, b: i32, s: Rect) -> BiggerRect;
 
     fn huge_struct(s: Huge) -> Huge;
+
+    fn float_point(p: FloatPoint) -> FloatPoint;
 }
 
 fn main() {
@@ -79,6 +88,7 @@ fn main() {
     let t = BiggerRect { s: s, a: 27834, b: 7657 };
     let u = FloatRect { a: 3489, b: 3490, c: 8. };
     let v = Huge { a: 5647, b: 5648, c: 5649, d: 5650, e: 5651 };
+    let p = FloatPoint { x: 5., y: -3. };
 
     unsafe {
         byval_rect(1, 2, 3, 4, 5, s);
@@ -94,5 +104,6 @@ fn main() {
         assert_eq!(split_ret_byval_struct(1, 2, s), s);
         assert_eq!(sret_byval_struct(1, 2, 3, 4, s), t);
         assert_eq!(sret_split_struct(1, 2, s), t);
+        assert_eq!(float_point(p), p);
     }
 }