Auto merge of #31333 - lambda:31273-abort-on-stack-overflow, r=brson

Abort on stack overflow instead of re-raising SIGSEGV

We use guard pages that cause the process to abort to protect against
undefined behavior in the event of stack overflow.  We have a handler
that catches segfaults, prints out an error message if the segfault was
due to a stack overflow, then unregisters itself and returns to allow
the signal to be re-raised and kill the process.

This caused some confusion, as it was unexpected that safe code would be
able to cause a segfault, while it's easy to overflow the stack in safe
code.  To avoid this confusion, when we detect a segfault in the guard
page, abort instead of the previous behavior of re-raising SIGSEGV.

To test this, we need to adapt the tests for segfault to actually check
the exit status.  Doing so revealed that the existing test for segfault
behavior was actually invalid; LLVM optimizes the explicit null pointer
reference down to an illegal instruction, so the program aborts with
SIGILL instead of SIGSEGV and the test didn't actually trigger the
signal handler at all.  Use a C helper function to get a null pointer
that LLVM can't optimize away, so we get our segfault instead.

This is a [breaking-change] if anyone is relying on the exact signal
raised to kill a process on stack overflow.

Closes #31273

2 files changed, 48 insertions, 3 deletions

diff --git a/src/test/run-pass/out-of-stack.rs b/src/test/run-pass/out-of-stack.rs
index 149de10ce02..4401861a163 100644
--- a/src/test/run-pass/out-of-stack.rs
+++ b/src/test/run-pass/out-of-stack.rs
@@ -12,6 +12,10 @@
 // ignore-musl
 
 #![feature(asm)]
+#![feature(libc)]
+
+#[cfg(unix)]
+extern crate libc;
 
 use std::env;
 use std::process::Command;
@@ -35,6 +39,24 @@ fn loud_recurse() {
     black_box(()); // don't optimize this into a tail call. please.
 }
 
+#[cfg(unix)]
+fn check_status(status: std::process::ExitStatus)
+{
+    use libc;
+    use std::os::unix::process::ExitStatusExt;
+
+    assert!(!status.success());
+    assert!(status.signal() != Some(libc::SIGSEGV)
+            && status.signal() != Some(libc::SIGBUS));
+}
+
+#[cfg(not(unix))]
+fn check_status(status: std::process::ExitStatus)
+{
+    assert!(!status.success());
+}
+
+
 fn main() {
     let args: Vec<String> = env::args().collect();
     if args.len() > 1 && args[1] == "silent" {
@@ -62,7 +84,9 @@ fn main() {
             println!("testing: {}", mode);
 
             let silent = Command::new(&args[0]).arg(mode).output().unwrap();
-            assert!(!silent.status.success());
+
+            check_status(silent.status);
+
             let error = String::from_utf8_lossy(&silent.stderr);
             assert!(error.contains("has overflowed its stack"),
                     "missing overflow message: {}", error);
diff --git a/src/test/run-pass/segfault-no-out-of-stack.rs b/src/test/run-pass/segfault-no-out-of-stack.rs
index 6f5fc8e6b4e..0158c4282da 100644
--- a/src/test/run-pass/segfault-no-out-of-stack.rs
+++ b/src/test/run-pass/segfault-no-out-of-stack.rs
@@ -15,10 +15,31 @@ extern crate libc;
 use std::process::{Command, ExitStatus};
 use std::env;
 
+#[link(name = "rust_test_helpers")]
+extern {
+    fn rust_get_null_ptr() -> *mut ::libc::c_char;
+}
+
+#[cfg(unix)]
+fn check_status(status: std::process::ExitStatus)
+{
+    use libc;
+    use std::os::unix::process::ExitStatusExt;
+
+    assert!(status.signal() == Some(libc::SIGSEGV)
+            || status.signal() == Some(libc::SIGBUS));
+}
+
+#[cfg(not(unix))]
+fn check_status(status: std::process::ExitStatus)
+{
+    assert!(!status.success());
+}
+
 fn main() {
     let args: Vec<String> = env::args().collect();
     if args.len() > 1 && args[1] == "segfault" {
-        unsafe { *(0 as *mut isize) = 1 }; // trigger a segfault
+        unsafe { *rust_get_null_ptr() = 1; }; // trigger a segfault
     } else {
         let segfault = Command::new(&args[0]).arg("segfault").output().unwrap();
         let stderr = String::from_utf8_lossy(&segfault.stderr);
@@ -26,7 +47,7 @@ fn main() {
         println!("stdout: {}", stdout);
         println!("stderr: {}", stderr);
         println!("status: {}", segfault.status);
-        assert!(!segfault.status.success());
+        check_status(segfault.status);
         assert!(!stderr.contains("has overflowed its stack"));
     }
 }