Auto merge of #86419 - ricobbe:raw-dylib-stdcall, r=petrochenkov

Add support for raw-dylib with stdcall, fastcall functions

Next stage of work for #58713: allow `extern "stdcall"` and `extern "fastcall"` with `#[link(kind = "raw-dylib")]`.

I've deliberately omitted support for vectorcall, as that doesn't currently work, and I wanted to get this out for review.  (I haven't really investigated the vectorcall failure much yet, but at first (very cursory) glance it appears that the problem is elsewhere.)

15 files changed, 293 insertions, 2 deletions

diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/Makefile b/src/test/run-make/raw-dylib-alt-calling-convention/Makefile
new file mode 100644
index 00000000000..0f874333fa0
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/Makefile
@@ -0,0 +1,18 @@
+# Test the behavior of #[link(.., kind = "raw-dylib")] with alternative calling conventions.
+
+# only-i686-pc-windows-msvc
+
+-include ../../run-make-fulldeps/tools.mk
+
+all:
+	$(call COMPILE_OBJ,"$(TMPDIR)"/extern.obj,extern.c)
+	$(CC) "$(TMPDIR)"/extern.obj -link -dll -out:"$(TMPDIR)"/extern.dll
+	$(RUSTC) --crate-type lib --crate-name raw_dylib_alt_calling_convention_test lib.rs
+	$(RUSTC) --crate-type bin driver.rs -L "$(TMPDIR)"
+	"$(TMPDIR)"/driver > "$(TMPDIR)"/output.txt
+
+ifdef RUSTC_BLESS_TEST
+	cp "$(TMPDIR)"/output.txt output.txt
+else
+	$(DIFF) output.txt "$(TMPDIR)"/output.txt
+endif
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs b/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs
new file mode 100644
index 00000000000..3710507f5e4
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/driver.rs
@@ -0,0 +1,5 @@
+extern crate raw_dylib_alt_calling_convention_test;
+
+fn main() {
+    raw_dylib_alt_calling_convention_test::library_function();
+}
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/extern.c b/src/test/run-make/raw-dylib-alt-calling-convention/extern.c
new file mode 100644
index 00000000000..8f64abf2fb5
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/extern.c
@@ -0,0 +1,123 @@
+#include <stdio.h>
+#include <stdint.h>
+
+struct S {
+    uint8_t x;
+    int32_t y;
+};
+
+struct S2 {
+    int32_t x;
+    uint8_t y;
+};
+
+struct S3 {
+    uint8_t x[5];
+};
+
+__declspec(dllexport) void __stdcall stdcall_fn_1(int i) {
+    printf("stdcall_fn_1(%d)\n", i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __stdcall stdcall_fn_2(uint8_t i, float f) {
+    printf("stdcall_fn_2(%d, %.1f)\n", i, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __stdcall stdcall_fn_3(double d) {
+    printf("stdcall_fn_3(%.1f)\n", d);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __stdcall stdcall_fn_4(uint8_t i, uint8_t j, float f) {
+    printf("stdcall_fn_4(%d, %d, %.1f)\n", i, j, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __stdcall stdcall_fn_5(struct S s, int i) {
+    printf("stdcall_fn_5(S { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+// Test that stdcall support works correctly with the nullable pointer optimization.
+__declspec(dllexport) void __stdcall stdcall_fn_6(struct S* s) {
+    if (s) {
+        printf("stdcall_fn_6(S { x: %d, y: %d })\n", s->x, s->y);
+    } else {
+        printf("stdcall_fn_6(null)\n");
+    }
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __stdcall stdcall_fn_7(struct S2 s, int i) {
+    printf("stdcall_fn_7(S2 { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+// Verify that we compute the correct amount of space in the argument list for a 5-byte struct.
+__declspec(dllexport) void __stdcall stdcall_fn_8(struct S3 s, struct S3 t) {
+    printf("stdcall_fn_8(S3 { x: [%d, %d, %d, %d, %d] }, S3 { x: [%d, %d, %d, %d, %d] })\n",
+           s.x[0], s.x[1], s.x[2], s.x[3], s.x[4],
+           t.x[0], t.x[1], t.x[2], t.x[3], t.x[4]
+        );
+    fflush(stdout);
+}
+
+// test whether f64/double values are aligned on 4-byte or 8-byte boundaries.
+__declspec(dllexport) void __stdcall stdcall_fn_9(uint8_t x, double y) {
+    printf("stdcall_fn_9(%d, %.1f)\n", x, y);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_1(int i) {
+    printf("fastcall_fn_1(%d)\n", i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_2(uint8_t i, float f) {
+    printf("fastcall_fn_2(%d, %.1f)\n", i, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_3(double d) {
+    printf("fastcall_fn_3(%.1f)\n", d);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_4(uint8_t i, uint8_t j, float f) {
+    printf("fastcall_fn_4(%d, %d, %.1f)\n", i, j, f);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_5(struct S s, int i) {
+    printf("fastcall_fn_5(S { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_6(struct S* s) {
+    if (s) {
+        printf("fastcall_fn_6(S { x: %d, y: %d })\n", s->x, s->y);
+    } else {
+        printf("fastcall_fn_6(null)\n");
+    }
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_7(struct S2 s, int i) {
+    printf("fastcall_fn_7(S2 { x: %d, y: %d }, %d)\n", s.x, s.y, i);
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_8(struct S3 s, struct S3 t) {
+    printf("fastcall_fn_8(S3 { x: [%d, %d, %d, %d, %d] }, S3 { x: [%d, %d, %d, %d, %d] })\n",
+           s.x[0], s.x[1], s.x[2], s.x[3], s.x[4],
+           t.x[0], t.x[1], t.x[2], t.x[3], t.x[4]
+        );
+    fflush(stdout);
+}
+
+__declspec(dllexport) void __fastcall fastcall_fn_9(uint8_t x, double y) {
+    printf("fastcall_fn_9(%d, %.1f)\n", x, y);
+    fflush(stdout);
+}
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs b/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs
new file mode 100644
index 00000000000..ba0f1418aba
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/lib.rs
@@ -0,0 +1,71 @@
+#![feature(raw_dylib)]
+
+#[repr(C)]
+#[derive(Clone)]
+struct S {
+    x: u8,
+    y: i32,
+}
+
+#[repr(C)]
+#[derive(Clone)]
+struct S2 {
+    x: i32,
+    y: u8,
+}
+
+#[repr(C)]
+#[derive(Clone)]
+struct S3 {
+    x: [u8; 5],
+}
+
+#[link(name = "extern", kind = "raw-dylib")]
+extern "stdcall" {
+    fn stdcall_fn_1(i: i32);
+    fn stdcall_fn_2(c: u8, f: f32);
+    fn stdcall_fn_3(d: f64);
+    fn stdcall_fn_4(i: u8, j: u8, f: f32);
+    fn stdcall_fn_5(a: S, b: i32);
+    fn stdcall_fn_6(a: Option<&S>);
+    fn stdcall_fn_7(a: S2, b: i32);
+    fn stdcall_fn_8(a: S3, b: S3);
+    fn stdcall_fn_9(x: u8, y: f64);
+}
+
+#[link(name = "extern", kind = "raw-dylib")]
+extern "fastcall" {
+    fn fastcall_fn_1(i: i32);
+    fn fastcall_fn_2(c: u8, f: f32);
+    fn fastcall_fn_3(d: f64);
+    fn fastcall_fn_4(i: u8, j: u8, f: f32);
+    fn fastcall_fn_5(a: S, b: i32);
+    fn fastcall_fn_6(a: Option<&S>);
+    fn fastcall_fn_7(a: S2, b: i32);
+    fn fastcall_fn_8(a: S3, b: S3);
+    fn fastcall_fn_9(x: u8, y: f64);
+}
+
+pub fn library_function() {
+    unsafe {
+        stdcall_fn_1(14);
+        stdcall_fn_2(16, 3.5);
+        stdcall_fn_3(3.5);
+        stdcall_fn_4(1, 2, 3.0);
+        stdcall_fn_5(S { x: 1, y: 2 }, 16);
+        stdcall_fn_6(Some(&S { x: 10, y: 12 }));
+        stdcall_fn_7(S2 { x: 15, y: 16 }, 3);
+        stdcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
+        stdcall_fn_9(1, 3.0);
+
+        fastcall_fn_1(14);
+        fastcall_fn_2(16, 3.5);
+        fastcall_fn_3(3.5);
+        fastcall_fn_4(1, 2, 3.0);
+        fastcall_fn_5(S { x: 1, y: 2 }, 16);
+        fastcall_fn_6(Some(&S { x: 10, y: 12 }));
+        fastcall_fn_7(S2 { x: 15, y: 16 }, 3);
+        fastcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] });
+        fastcall_fn_9(1, 3.0);
+    }
+}
diff --git a/src/test/run-make/raw-dylib-alt-calling-convention/output.txt b/src/test/run-make/raw-dylib-alt-calling-convention/output.txt
new file mode 100644
index 00000000000..be598a22027
--- /dev/null
+++ b/src/test/run-make/raw-dylib-alt-calling-convention/output.txt
@@ -0,0 +1,18 @@
+stdcall_fn_1(14)
+stdcall_fn_2(16, 3.5)
+stdcall_fn_3(3.5)
+stdcall_fn_4(1, 2, 3.0)
+stdcall_fn_5(S { x: 1, y: 2 }, 16)
+stdcall_fn_6(S { x: 10, y: 12 })
+stdcall_fn_7(S2 { x: 15, y: 16 }, 3)
+stdcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] })
+stdcall_fn_9(1, 3.0)
+fastcall_fn_1(14)
+fastcall_fn_2(16, 3.5)
+fastcall_fn_3(3.5)
+fastcall_fn_4(1, 2, 3.0)
+fastcall_fn_5(S { x: 1, y: 2 }, 16)
+fastcall_fn_6(S { x: 10, y: 12 })
+fastcall_fn_7(S2 { x: 15, y: 16 }, 3)
+fastcall_fn_8(S3 { x: [1, 2, 3, 4, 5] }, S3 { x: [6, 7, 8, 9, 10] })
+fastcall_fn_9(1, 3.0)
diff --git a/src/test/run-make/raw-dylib/Makefile b/src/test/run-make/raw-dylib-c/Makefile
index 7ce46fd9331..26ab4d34764 100644
--- a/src/test/run-make/raw-dylib/Makefile
+++ b/src/test/run-make/raw-dylib-c/Makefile
@@ -1,7 +1,6 @@
 # Test the behavior of #[link(.., kind = "raw-dylib")] on windows-msvc
 
-# only-windows
-# only-msvc
+# only-windows-msvc
 
 -include ../../run-make-fulldeps/tools.mk
 
diff --git a/src/test/run-make/raw-dylib/driver.rs b/src/test/run-make/raw-dylib-c/driver.rs
index 4059ede11fc..4059ede11fc 100644
--- a/src/test/run-make/raw-dylib/driver.rs
+++ b/src/test/run-make/raw-dylib-c/driver.rs
diff --git a/src/test/run-make/raw-dylib/extern_1.c b/src/test/run-make/raw-dylib-c/extern_1.c
index 72737c086eb..72737c086eb 100644
--- a/src/test/run-make/raw-dylib/extern_1.c
+++ b/src/test/run-make/raw-dylib-c/extern_1.c
diff --git a/src/test/run-make/raw-dylib/extern_2.c b/src/test/run-make/raw-dylib-c/extern_2.c
index ae87fc3f821..ae87fc3f821 100644
--- a/src/test/run-make/raw-dylib/extern_2.c
+++ b/src/test/run-make/raw-dylib-c/extern_2.c
diff --git a/src/test/run-make/raw-dylib/lib.rs b/src/test/run-make/raw-dylib-c/lib.rs
index d8e6301f38e..d8e6301f38e 100644
--- a/src/test/run-make/raw-dylib/lib.rs
+++ b/src/test/run-make/raw-dylib-c/lib.rs
diff --git a/src/test/run-make/raw-dylib/output.txt b/src/test/run-make/raw-dylib-c/output.txt
index 7800cba1872..7800cba1872 100644
--- a/src/test/run-make/raw-dylib/output.txt
+++ b/src/test/run-make/raw-dylib-c/output.txt
diff --git a/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.rs b/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.rs
new file mode 100644
index 00000000000..3f7597498ba
--- /dev/null
+++ b/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.rs
@@ -0,0 +1,19 @@
+// only-i686-pc-windows-msvc
+// compile-flags: --crate-type lib --emit link
+#![allow(clashing_extern_declarations)]
+#![feature(raw_dylib)]
+//~^ WARN the feature `raw_dylib` is incomplete
+#[link(name = "foo", kind = "raw-dylib")]
+extern "C" {
+    fn f(x: i32);
+    //~^ ERROR multiple definitions of external function `f` from library `foo.dll` have different calling conventions
+}
+
+pub fn lib_main() {
+    #[link(name = "foo", kind = "raw-dylib")]
+    extern "stdcall" {
+        fn f(x: i32);
+    }
+
+    unsafe { f(42); }
+}
diff --git a/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.stderr b/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.stderr
new file mode 100644
index 00000000000..91f6f0cf722
--- /dev/null
+++ b/src/test/ui/rfc-2627-raw-dylib/multiple-definitions.stderr
@@ -0,0 +1,17 @@
+warning: the feature `raw_dylib` is incomplete and may not be safe to use and/or cause compiler crashes
+  --> $DIR/multiple-definitions.rs:4:12
+   |
+LL | #![feature(raw_dylib)]
+   |            ^^^^^^^^^
+   |
+   = note: `#[warn(incomplete_features)]` on by default
+   = note: see issue #58713 <https://github.com/rust-lang/rust/issues/58713> for more information
+
+error: multiple definitions of external function `f` from library `foo.dll` have different calling conventions
+  --> $DIR/multiple-definitions.rs:8:5
+   |
+LL |     fn f(x: i32);
+   |     ^^^^^^^^^^^^^
+
+error: aborting due to previous error; 1 warning emitted
+
diff --git a/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.rs b/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.rs
new file mode 100644
index 00000000000..e5a5ac2eb2b
--- /dev/null
+++ b/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.rs
@@ -0,0 +1,13 @@
+// only-x86_64-pc-windows-msvc
+// compile-flags: --crate-type lib --emit link
+#![allow(incomplete_features)]
+#![feature(raw_dylib)]
+#[link(name = "foo", kind = "raw-dylib")]
+extern "stdcall" {
+    fn f(x: i32);
+    //~^ ERROR ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
+}
+
+pub fn lib_main() {
+    unsafe { f(42); }
+}
diff --git a/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.stderr b/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.stderr
new file mode 100644
index 00000000000..fc9008128ae
--- /dev/null
+++ b/src/test/ui/rfc-2627-raw-dylib/unsupported-abi.stderr
@@ -0,0 +1,8 @@
+error: ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
+  --> $DIR/unsupported-abi.rs:7:5
+   |
+LL |     fn f(x: i32);
+   |     ^^^^^^^^^^^^^
+
+error: aborting due to previous error
+