Auto merge of #99476 - dpaoliello:rawdylibvectorcall, r=michaelwoerister

Add tests for raw-dylib with vectorcall, and fix vectorcall code generation

* Adds tests for using `raw-dylib` (#58713) with `vectorcall`.
* Fixed code generation for `vectorcall` (parameters have to be marked with `InReg`, just like `fastcall`).
* Enabled running the `raw-dylib` `fastcall` tests when using MSVC (since I had to add support in the test for running MSVC-only tests since GCC doesn't support `vectorcall`).

8 files changed, 144 insertions, 30 deletions

diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index c33219e4700..9f6079ecba4 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -472,7 +472,9 @@ impl<'tcx> Collector<'tcx> {
                 Abi::Fastcall { .. } => {
                     DllCallingConvention::Fastcall(self.i686_arg_list_size(item))
                 }
-                // Vectorcall is intentionally not supported at this time.
+                Abi::Vectorcall { .. } => {
+                    DllCallingConvention::Vectorcall(self.i686_arg_list_size(item))
+                }
                 _ => {
                     self.tcx.sess.span_fatal(
                         item.span,
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index ca1d1302ec6..577126a95cc 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -669,8 +669,10 @@ impl<'a, Ty> FnAbi<'a, Ty> {
 
         match &cx.target_spec().arch[..] {
             "x86" => {
-                let flavor = if let spec::abi::Abi::Fastcall { .. } = abi {
-                    x86::Flavor::Fastcall
+                let flavor = if let spec::abi::Abi::Fastcall { .. }
+                | spec::abi::Abi::Vectorcall { .. } = abi
+                {
+                    x86::Flavor::FastcallOrVectorcall
                 } else {
                     x86::Flavor::General
                 };
diff --git a/compiler/rustc_target/src/abi/call/x86.rs b/compiler/rustc_target/src/abi/call/x86.rs
index d169087dfbd..c7d59baf919 100644
--- a/compiler/rustc_target/src/abi/call/x86.rs
+++ b/compiler/rustc_target/src/abi/call/x86.rs
@@ -5,7 +5,7 @@ use crate::spec::HasTargetSpec;
 #[derive(PartialEq)]
 pub enum Flavor {
     General,
-    Fastcall,
+    FastcallOrVectorcall,
 }
 
 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, flavor: Flavor)
@@ -60,9 +60,9 @@ where
         }
     }
 
-    if flavor == Flavor::Fastcall {
+    if flavor == Flavor::FastcallOrVectorcall {
         // Mark arguments as InReg like clang does it,
-        // so our fastcall is compatible with C/C++ fastcall.
+        // so our fastcall/vectorcall is compatible with C/C++ fastcall/vectorcall.
 
         // Clang reference: lib/CodeGen/TargetInfo.cpp
         // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/Makefile b/src/test/run-make/raw-dylib-alt-calling-convention/Makefile
index 1badde54112..a254285ab76 100644
--- a/src/test/run-make/raw-dylib-alt-calling-convention/Makefile
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/Makefile
@@ -14,10 +14,19 @@ ifdef IS_MSVC
 else
 	$(CC) "$(TMPDIR)"/extern.obj -shared -o "$(TMPDIR)"/extern.dll
 endif
-	"$(TMPDIR)"/driver > "$(TMPDIR)"/output.txt
 
+	"$(TMPDIR)"/driver > "$(TMPDIR)"/output.txt
 ifdef RUSTC_BLESS_TEST
 	cp "$(TMPDIR)"/output.txt output.txt
 else
 	$(DIFF) output.txt "$(TMPDIR)"/output.txt
 endif
+
+ifdef IS_MSVC
+	"$(TMPDIR)"/driver true > "$(TMPDIR)"/output.msvc.txt
+ifdef RUSTC_BLESS_TEST
+	cp "$(TMPDIR)"/output.msvc.txt output.msvc.txt
+else
+	$(DIFF) output.msvc.txt "$(TMPDIR)"/output.msvc.txt
+endif
+endif
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs b/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs
index 3710507f5e4..b7f372c6b2b 100644
--- a/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs
@@ -1,5 +1,8 @@
 extern crate raw_dylib_alt_calling_convention_test;
 
 fn main() {
-    raw_dylib_alt_calling_convention_test::library_function();
+    raw_dylib_alt_calling_convention_test::library_function(
+        std::env::args().skip(1).next().map_or(
+            false,
+            |s| std::str::FromStr::from_str(&s).unwrap()));
 }
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/extern.c b/src/test/run-make/raw-dylib-alt-calling-convention/extern.c
index 8f64abf2fb5..0c4d12af9b2 100644
--- a/src/test/run-make/raw-dylib-alt-calling-convention/extern.c
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/extern.c
@@ -121,3 +121,58 @@ __declspec(dllexport) void __fastcall fastcall_fn_9(uint8_t x, double y) {
     printf("fastcall_fn_9(%d, %.1f)\n", x, y);
     fflush(stdout);
 }
+
+// GCC doesn't support vectorcall: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89485
+#ifdef _MSC_VER
+__declspec(dllexport) void __vectorcall vectorcall_fn_1(int i) {
+    printf("vectorcall_fn_1(%d)\n", i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_2(uint8_t i, float f) {
+    printf("vectorcall_fn_2(%d, %.1f)\n", i, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_3(double d) {
+    printf("vectorcall_fn_3(%.1f)\n", d);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_4(uint8_t i, uint8_t j, float f) {
+    printf("vectorcall_fn_4(%d, %d, %.1f)\n", i, j, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_5(struct S s, int i) {
+    printf("vectorcall_fn_5(S { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_6(struct S* s) {
+    if (s) {
+        printf("vectorcall_fn_6(S { x: %d, y: %d })\n", s->x, s->y);
+    } else {
+        printf("vectorcall_fn_6(null)\n");
+    }
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_7(struct S2 s, int i) {
+    printf("vectorcall_fn_7(S2 { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_8(struct S3 s, struct S3 t) {
+    printf("vectorcall_fn_8(S3 { x: [%d, %d, %d, %d, %d] }, S3 { x: [%d, %d, %d, %d, %d] })\n",
+           s.x[0], s.x[1], s.x[2], s.x[3], s.x[4],
+           t.x[0], t.x[1], t.x[2], t.x[3], t.x[4]
+        );
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __vectorcall vectorcall_fn_9(uint8_t x, double y) {
+    printf("vectorcall_fn_9(%d, %.1f)\n", x, y);
+    fflush(stdout);
+}
+#endif
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs b/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs
index 165792b0490..b5e9415b2be 100644
--- a/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs
@@ -1,4 +1,5 @@
 #![feature(raw_dylib)]
+#![feature(abi_vectorcall)]
 
 #[repr(C)]
 #[derive(Clone)]
@@ -46,29 +47,60 @@ extern "fastcall" {
     fn fastcall_fn_9(x: u8, y: f64);
 }
 
-pub fn library_function() {
+#[cfg(target_env = "msvc")]
+#[link(name = "extern", kind = "raw-dylib")]
+extern "vectorcall" {
+    fn vectorcall_fn_1(i: i32);
+    fn vectorcall_fn_2(c: u8, f: f32);
+    fn vectorcall_fn_3(d: f64);
+    fn vectorcall_fn_4(i: u8, j: u8, f: f32);
+    fn vectorcall_fn_5(a: S, b: i32);
+    fn vectorcall_fn_6(a: Option<&S>);
+    fn vectorcall_fn_7(a: S2, b: i32);
+    fn vectorcall_fn_8(a: S3, b: S3);
+    fn vectorcall_fn_9(x: u8, y: f64);
+}
+
+pub fn library_function(run_msvc_only: bool) {
     unsafe {
-        stdcall_fn_1(14);
-        stdcall_fn_2(16, 3.5);
-        stdcall_fn_3(3.5);
-        stdcall_fn_4(1, 2, 3.0);
-        stdcall_fn_5(S { x: 1, y: 2 }, 16);
-        stdcall_fn_6(Some(&S { x: 10, y: 12 }));
-        stdcall_fn_7(S2 { x: 15, y: 16 }, 3);
-        stdcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
-        stdcall_fn_9(1, 3.0);
+        if !run_msvc_only {
+            stdcall_fn_1(14);
+            stdcall_fn_2(16, 3.5);
+            stdcall_fn_3(3.5);
+            stdcall_fn_4(1, 2, 3.0);
+            stdcall_fn_5(S { x: 1, y: 2 }, 16);
+            stdcall_fn_6(Some(&S { x: 10, y: 12 }));
+            stdcall_fn_7(S2 { x: 15, y: 16 }, 3);
+            stdcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
+            stdcall_fn_9(1, 3.0);
+
+            fastcall_fn_1(14);
+            fastcall_fn_2(16, 3.5);
+            fastcall_fn_3(3.5);
+            fastcall_fn_4(1, 2, 3.0);
+            fastcall_fn_6(Some(&S { x: 10, y: 12 }));
+            fastcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
+            fastcall_fn_9(1, 3.0);
+        } else {
+            // FIXME: 91167
+            // rustc generates incorrect code for the calls to fastcall_fn_5 and fastcall_fn_7
+            // on i686-pc-windows-gnu; disabling these until the indicated issue is fixed.
+            fastcall_fn_5(S { x: 1, y: 2 }, 16);
+            fastcall_fn_7(S2 { x: 15, y: 16 }, 3);
 
-        fastcall_fn_1(14);
-        fastcall_fn_2(16, 3.5);
-        fastcall_fn_3(3.5);
-        fastcall_fn_4(1, 2, 3.0);
-        // FIXME: 91167
-        // rustc generates incorrect code for the calls to fastcall_fn_5 and fastcall_fn_7
-        // on i686-pc-windows-gnu; commenting these out until the indicated issue is fixed.
-        //fastcall_fn_5(S { x: 1, y: 2 }, 16);
-        fastcall_fn_6(Some(&S { x: 10, y: 12 }));
-        //fastcall_fn_7(S2 { x: 15, y: 16 }, 3);
-        fastcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
-        fastcall_fn_9(1, 3.0);
+            // GCC doesn't support vectorcall: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89485
+            #[cfg(target_env = "msvc")]
+            {
+                vectorcall_fn_1(14);
+                vectorcall_fn_2(16, 3.5);
+                vectorcall_fn_3(3.5);
+                vectorcall_fn_4(1, 2, 3.0);
+                vectorcall_fn_5(S { x: 1, y: 2 }, 16);
+                vectorcall_fn_6(Some(&S { x: 10, y: 12 }));
+                vectorcall_fn_7(S2 { x: 15, y: 16 }, 3);
+                vectorcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
+                vectorcall_fn_9(1, 3.0);
+            }
+        }
     }
 }
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/output.msvc.txt b/src/test/run-make/raw-dylib-alt-calling-convention/output.msvc.txt
new file mode 100644
index 00000000000..9ddd1b11016
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/output.msvc.txt
@@ -0,0 +1,11 @@
+fastcall_fn_5(S { x: 1, y: 2 }, 16)
+fastcall_fn_7(S2 { x: 15, y: 16 }, 3)
+vectorcall_fn_1(14)
+vectorcall_fn_2(16, 3.5)
+vectorcall_fn_3(3.5)
+vectorcall_fn_4(1, 2, 3.0)
+vectorcall_fn_5(S { x: 1, y: 2 }, 16)
+vectorcall_fn_6(S { x: 10, y: 12 })
+vectorcall_fn_7(S2 { x: 15, y: 16 }, 3)
+vectorcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] })
+vectorcall_fn_9(1, 3.0)