Rollup merge of #122847 - workingjubilee:suggest-rust-min-stack-workaround-on-overflow, r=TaKO8Ki

Suggest `RUST_MIN_STACK` workaround on overflow

For some Rust crates, like p384, we can't do a whole lot about it even if the stack overflow is reported like in rust-lang/rust#122357 because the problem may be inside LLVM or another codegen backend. We can, however, suggest people set a new `RUST_MIN_STACK` value while handling the SIGSEGV, as that stack-setting will carry forward into the dylib.

As a bonus, this also leads to cleaning up the stack-setting code a bit.

2 files changed, 23 insertions, 16 deletions

diff --git a/compiler/rustc_driver_impl/src/signal_handler.rs b/compiler/rustc_driver_impl/src/signal_handler.rs
index deca1082221..441219eec90 100644
--- a/compiler/rustc_driver_impl/src/signal_handler.rs
+++ b/compiler/rustc_driver_impl/src/signal_handler.rs
@@ -1,6 +1,7 @@
 //! Signal handler for rustc
 //! Primarily used to extract a backtrace from stack overflow
 
+use rustc_interface::util::{DEFAULT_STACK_SIZE, STACK_SIZE};
 use std::alloc::{alloc, Layout};
 use std::{fmt, mem, ptr};
 
@@ -100,7 +101,10 @@ extern "C" fn print_stack_trace(_: libc::c_int) {
         written += 1;
     }
     raw_errln!("note: we would appreciate a report at https://github.com/rust-lang/rust");
-    written += 1;
+    // get the current stack size WITHOUT blocking and double it
+    let new_size = STACK_SIZE.get().copied().unwrap_or(DEFAULT_STACK_SIZE) * 2;
+    raw_errln!("help: you can increase rustc's stack size by setting RUST_MIN_STACK={new_size}");
+    written += 2;
     if written > 24 {
         // We probably just scrolled the earlier "we got SIGSEGV" message off the terminal
         raw_errln!("note: backtrace dumped due to SIGSEGV! resuming signal");
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 7d48f90db36..d09f8d7d7cf 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -48,12 +48,20 @@ pub fn add_configuration(cfg: &mut Cfg, sess: &mut Session, codegen_backend: &dy
     }
 }
 
-const STACK_SIZE: usize = 8 * 1024 * 1024;
-
-fn get_stack_size() -> Option<usize> {
-    // FIXME: Hacks on hacks. If the env is trying to override the stack size
-    // then *don't* set it explicitly.
-    env::var_os("RUST_MIN_STACK").is_none().then_some(STACK_SIZE)
+pub static STACK_SIZE: OnceLock<usize> = OnceLock::new();
+pub const DEFAULT_STACK_SIZE: usize = 8 * 1024 * 1024;
+
+fn init_stack_size() -> usize {
+    // Obey the environment setting or default
+    *STACK_SIZE.get_or_init(|| {
+        env::var_os("RUST_MIN_STACK")
+            .map(|os_str| os_str.to_string_lossy().into_owned())
+            // ignore if it is set to nothing
+            .filter(|s| s.trim() != "")
+            .map(|s| s.trim().parse::<usize>().unwrap())
+            // otherwise pick a consistent default
+            .unwrap_or(DEFAULT_STACK_SIZE)
+    })
 }
 
 pub(crate) fn run_in_thread_with_globals<F: FnOnce() -> R + Send, R: Send>(
@@ -66,10 +74,7 @@ pub(crate) fn run_in_thread_with_globals<F: FnOnce() -> R + Send, R: Send>(
     // the parallel compiler, in particular to ensure there is no accidental
     // sharing of data between the main thread and the compilation thread
     // (which might cause problems for the parallel compiler).
-    let mut builder = thread::Builder::new().name("rustc".to_string());
-    if let Some(size) = get_stack_size() {
-        builder = builder.stack_size(size);
-    }
+    let builder = thread::Builder::new().name("rustc".to_string()).stack_size(init_stack_size());
 
     // We build the session globals and run `f` on the spawned thread, because
     // `SessionGlobals` does not impl `Send` in the non-parallel compiler.
@@ -120,7 +125,7 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
         });
     }
 
-    let mut builder = rayon::ThreadPoolBuilder::new()
+    let builder = rayon::ThreadPoolBuilder::new()
         .thread_name(|_| "rustc".to_string())
         .acquire_thread_handler(jobserver::acquire_thread)
         .release_thread_handler(jobserver::release_thread)
@@ -144,10 +149,8 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
                     on_panic.disable();
                 })
                 .unwrap();
-        });
-    if let Some(size) = get_stack_size() {
-        builder = builder.stack_size(size);
-    }
+        })
+        .stack_size(init_stack_size());
 
     // We create the session globals on the main thread, then create the thread
     // pool. Upon creation, each worker thread created gets a copy of the