Enable inline stack probes on PowerPC and SystemZ

23 files changed, 107 insertions, 48 deletions

diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
index 803453c4ac4..9f4cc3d80f1 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_freebsd.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-freebsd".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
index 1cb9ce40cb1..21955a616c2 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_gnu.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
index 159335eb607..bbf86e4ff41 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_linux_musl.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/powerpc64_unknown_openbsd.rs
index 9cb3a67dc58..bfa61a21fb3 100644
--- a/compiler/rustc_target/src/spec/powerpc64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_unknown_openbsd.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-openbsd".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
index b7420d232ca..4ebf342ad22 100644
--- a/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/powerpc64_wrs_vxworks.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.cpu = "ppc64".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64le_unknown_freebsd.rs b/compiler/rustc_target/src/spec/powerpc64le_unknown_freebsd.rs
index a3d18004371..a7ab9078531 100644
--- a/compiler/rustc_target/src/spec/powerpc64le_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc64le_unknown_freebsd.rs
@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-freebsd".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
index e18ff3be448..69fd6be6dc0 100644
--- a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_gnu.rs
@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
index b84943d23a9..ae3a8b54519 100644
--- a/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc64le_unknown_linux_musl.rs
@@ -1,10 +1,11 @@
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.cpu = "ppc64le".into();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m64"]);
     base.max_atomic_width = Some(64);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc64le-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_freebsd.rs b/compiler/rustc_target/src/spec/powerpc_unknown_freebsd.rs
index 75ac66c276d..b5d4e5de05e 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_freebsd.rs
@@ -1,11 +1,12 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::freebsd_base::opts();
     // Extra hint to linker that we are generating secure-PLT code.
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "--target=powerpc-unknown-freebsd13.0"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-freebsd13.0".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
index 6686a0bbf04..0ceb66c327b 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnu.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
index 6a250f4b51c..716090f39ca 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_gnuspe.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mspe"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnuspe".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
index 34200c67906..d8cd158584a 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_linux_musl.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs b/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
index 60661ef9b5d..7053e4b9c26 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_netbsd.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-netbsd".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs b/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs
index ad2c3d40f35..dec85f9961b 100644
--- a/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/powerpc_unknown_openbsd.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::openbsd_base::opts();
     base.endian = Endian::Big;
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-openbsd".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
index 3f24966e06e..e0c5db6eacf 100644
--- a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-m32", "--secure-plt"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
index 0f04f41f9e5..c7f41b1da87 100644
--- a/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
+++ b/compiler/rustc_target/src/spec/powerpc_wrs_vxworks_spe.rs
@@ -1,10 +1,11 @@
 use crate::abi::Endian;
-use crate::spec::{LinkerFlavor, Target, TargetOptions};
+use crate::spec::{LinkerFlavor, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::vxworks_base::opts();
     base.add_pre_link_args(LinkerFlavor::Gcc, &["-mspe", "--secure-plt"]);
     base.max_atomic_width = Some(32);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "powerpc-unknown-linux-gnuspe".into(),
diff --git a/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
index 8757bbed8ad..9bb9c931f5c 100644
--- a/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
@@ -12,6 +12,7 @@ pub fn target() -> Target {
     base.features = "-vector".into();
     base.max_atomic_width = Some(64);
     base.min_global_align = Some(16);
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "s390x-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
index 4c855271a2a..f877279781d 100644
--- a/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::Target;
+use crate::spec::{StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
@@ -13,6 +13,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.min_global_align = Some(16);
     base.static_position_independent_executables = true;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "s390x-unknown-linux-musl".into(),
diff --git a/src/test/codegen/stack-probes-call.rs b/src/test/codegen/stack-probes-call.rs
new file mode 100644
index 00000000000..56b02fdecad
--- /dev/null
+++ b/src/test/codegen/stack-probes-call.rs
@@ -0,0 +1,22 @@
+// Check the "probe-stack" attribute for targets with `StackProbeType::Call`,
+// or `StackProbeType::InlineOrCall` when running on older LLVM.
+
+// compile-flags: -C no-prepopulate-passes
+// revisions: i686 x86_64
+//[i686] compile-flags: --target i686-unknown-linux-gnu
+//[i686] needs-llvm-components: x86
+//[x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//[x86_64] needs-llvm-components: x86
+
+#![crate_type = "rlib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="__rust_probestack"{{.*}} }
+}
diff --git a/src/test/codegen/stack-probes-inline.rs b/src/test/codegen/stack-probes-inline.rs
new file mode 100644
index 00000000000..837a1610810
--- /dev/null
+++ b/src/test/codegen/stack-probes-inline.rs
@@ -0,0 +1,26 @@
+// Check the "probe-stack" attribute for targets with `StackProbeType::Inline`,
+// or `StackProbeType::InlineOrCall` when running on newer LLVM.
+
+// compile-flags: -C no-prepopulate-passes
+// revisions: powerpc powerpc64 powerpc64le s390x
+//[powerpc] compile-flags: --target powerpc-unknown-linux-gnu
+//[powerpc] needs-llvm-components: powerpc
+//[powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//[powerpc64] needs-llvm-components: powerpc
+//[powerpc64le] compile-flags: --target powerpc64le-unknown-linux-gnu
+//[powerpc64le] needs-llvm-components: powerpc
+//[s390x] compile-flags: --target s390x-unknown-linux-gnu
+//[s390x] needs-llvm-components: systemz
+
+#![crate_type = "rlib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+#[lang = "sized"]
+trait Sized {}
+
+#[no_mangle]
+pub fn foo() {
+// CHECK: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}}"probe-stack"="inline-asm"{{.*}} }
+}
diff --git a/src/test/codegen/stack-probes.rs b/src/test/codegen/stack-probes.rs
deleted file mode 100644
index 9bd351df3ea..00000000000
--- a/src/test/codegen/stack-probes.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-// ignore-arm
-// ignore-aarch64
-// ignore-mips
-// ignore-mips64
-// ignore-powerpc
-// ignore-powerpc64
-// ignore-powerpc64le
-// ignore-riscv64
-// ignore-s390x
-// ignore-sparc
-// ignore-sparc64
-// ignore-wasm
-// ignore-emscripten
-// ignore-windows
-// compile-flags: -C no-prepopulate-passes
-
-#![crate_type = "lib"]
-
-#[no_mangle]
-pub fn foo() {
-// CHECK: @foo() unnamed_addr #0
-}
diff --git a/src/test/ui/abi/stack-probes-lto.rs b/src/test/ui/abi/stack-probes-lto.rs
index 74b5e843f77..6d934538f4c 100644
--- a/src/test/ui/abi/stack-probes-lto.rs
+++ b/src/test/ui/abi/stack-probes-lto.rs
@@ -3,8 +3,6 @@
 // ignore-aarch64
 // ignore-mips
 // ignore-mips64
-// ignore-powerpc
-// ignore-s390x
 // ignore-sparc
 // ignore-sparc64
 // ignore-wasm
diff --git a/src/test/ui/abi/stack-probes.rs b/src/test/ui/abi/stack-probes.rs
index b497af7abad..e7b91644b3b 100644
--- a/src/test/ui/abi/stack-probes.rs
+++ b/src/test/ui/abi/stack-probes.rs
@@ -3,8 +3,6 @@
 // ignore-aarch64
 // ignore-mips
 // ignore-mips64
-// ignore-powerpc
-// ignore-s390x
 // ignore-sparc
 // ignore-sparc64
 // ignore-wasm
@@ -27,8 +25,9 @@ fn main() {
     let args = env::args().skip(1).collect::<Vec<_>>();
     if args.len() > 0 {
         match &args[0][..] {
-            "main-thread" => recurse(&MaybeUninit::uninit()),
-            "child-thread" => thread::spawn(|| recurse(&MaybeUninit::uninit())).join().unwrap(),
+            "main-recurse" => overflow_recurse(),
+            "child-recurse" => thread::spawn(overflow_recurse).join().unwrap(),
+            "child-frame" => overflow_frame(),
             _ => panic!(),
         }
         return;
@@ -41,9 +40,10 @@ fn main() {
     // that we report stack overflow on the main thread, see #43052 for some
     // details
     if cfg!(not(target_os = "linux")) {
-        assert_overflow(Command::new(&me).arg("main-thread"));
+        assert_overflow(Command::new(&me).arg("main-recurse"));
     }
-    assert_overflow(Command::new(&me).arg("child-thread"));
+    assert_overflow(Command::new(&me).arg("child-recurse"));
+    assert_overflow(Command::new(&me).arg("child-frame"));
 }
 
 #[allow(unconditional_recursion)]
@@ -55,6 +55,23 @@ fn recurse(array: &MaybeUninit<[u64; 1024]>) {
     recurse(&local);
 }
 
+#[inline(never)]
+fn overflow_recurse() {
+    recurse(&MaybeUninit::uninit());
+}
+
+fn overflow_frame() {
+    // By using a 1MiB stack frame with only 512KiB stack, we'll jump over any
+    // guard page, even with 64K pages -- but stack probes should catch it.
+    const STACK_SIZE: usize = 512 * 1024;
+    thread::Builder::new().stack_size(STACK_SIZE).spawn(|| {
+        let local: MaybeUninit<[u8; 2 * STACK_SIZE]> = MaybeUninit::uninit();
+        unsafe {
+            black_box(local.as_ptr() as u64);
+        }
+    }).unwrap().join().unwrap();
+}
+
 fn assert_overflow(cmd: &mut Command) {
     let output = cmd.output().unwrap();
     assert!(!output.status.success());