Auto merge of #28631 - ranma42:robust-panic, r=alexcrichton

This is mainly to avoid infinite recursion and make debugging more convenient in the anomalous case in which `on_panic` panics.
I encountered such issues while changing libstd to debug/fix part of #28129.

While writing this I was wondering about which functions belong to `panicking` and which to `unwind`.
I placed them in this way mostly because of convenience, but I would strongly appreciate guidance.

2 files changed, 54 insertions, 41 deletions

diff --git a/src/libstd/panicking.rs b/src/libstd/panicking.rs
index 9715939d644..2b2af350c99 100644
--- a/src/libstd/panicking.rs
+++ b/src/libstd/panicking.rs
@@ -12,11 +12,15 @@ use prelude::v1::*;
 use io::prelude::*;
 
 use any::Any;
+use cell::Cell;
 use cell::RefCell;
+use intrinsics;
 use sys::stdio::Stderr;
 use sys_common::backtrace;
 use sys_common::thread_info;
-use sys_common::unwind;
+use sys_common::util;
+
+thread_local! { pub static PANIC_COUNT: Cell<usize> = Cell::new(0) }
 
 thread_local! {
     pub static LOCAL_STDERR: RefCell<Option<Box<Write + Send>>> = {
@@ -24,7 +28,8 @@ thread_local! {
     }
 }
 
-pub fn on_panic(obj: &(Any+Send), file: &'static str, line: u32) {
+fn log_panic(obj: &(Any+Send), file: &'static str, line: u32,
+             log_backtrace: bool) {
     let msg = match obj.downcast_ref::<&'static str>() {
         Some(s) => *s,
         None => match obj.downcast_ref::<String>() {
@@ -35,37 +40,59 @@ pub fn on_panic(obj: &(Any+Send), file: &'static str, line: u32) {
     let mut err = Stderr::new().ok();
     let thread = thread_info::current_thread();
     let name = thread.as_ref().and_then(|t| t.name()).unwrap_or("<unnamed>");
+
+    let write = |err: &mut ::io::Write| {
+        let _ = writeln!(err, "thread '{}' panicked at '{}', {}:{}",
+                         name, msg, file, line);
+        if log_backtrace {
+            let _ = backtrace::write(err);
+        }
+    };
+
     let prev = LOCAL_STDERR.with(|s| s.borrow_mut().take());
     match (prev, err.as_mut()) {
         (Some(mut stderr), _) => {
-            // FIXME: what to do when the thread printing panics?
-            let _ = writeln!(stderr,
-                             "thread '{}' panicked at '{}', {}:{}\n",
-                             name, msg, file, line);
-            if backtrace::log_enabled() {
-                let _ = backtrace::write(&mut *stderr);
-            }
+            write(&mut *stderr);
             let mut s = Some(stderr);
             LOCAL_STDERR.with(|slot| {
                 *slot.borrow_mut() = s.take();
             });
         }
-        (None, Some(ref mut err)) => {
-            let _ = writeln!(err, "thread '{}' panicked at '{}', {}:{}",
-                             name, msg, file, line);
-            if backtrace::log_enabled() {
-                let _ = backtrace::write(err);
-            }
-        }
+        (None, Some(ref mut err)) => { write(err) }
         _ => {}
     }
+}
 
-    // If this is a double panic, make sure that we printed a backtrace
-    // for this panic.
-    match err {
-        Some(ref mut err) if unwind::panicking() && !backtrace::log_enabled() => {
-            let _ = backtrace::write(err);
-        }
-        _ => {}
+pub fn on_panic(obj: &(Any+Send), file: &'static str, line: u32) {
+    let panics = PANIC_COUNT.with(|s| {
+        let count = s.get() + 1;
+        s.set(count);
+        count
+    });
+
+    // If this is the third nested call, on_panic triggered the last panic,
+    // otherwise the double-panic check would have aborted the process.
+    // Even if it is likely that on_panic was unable to log the backtrace,
+    // abort immediately to avoid infinite recursion, so that attaching a
+    // debugger provides a useable stacktrace.
+    if panics >= 3 {
+        util::dumb_print(format_args!("thread panicked while processing \
+                                       panic. aborting."));
+        unsafe { intrinsics::abort() }
+    }
+
+    // If this is a double panic, make sure that we print a backtrace
+    // for this panic. Otherwise only print it if logging is enabled.
+    let log_backtrace = panics >= 2 || backtrace::log_enabled();
+    log_panic(obj, file, line, log_backtrace);
+
+    if panics >= 2 {
+        // If a thread panics while it's already unwinding then we
+        // have limited options. Currently our preference is to
+        // just abort. In the future we may consider resuming
+        // unwinding or otherwise exiting the thread cleanly.
+        util::dumb_print(format_args!("thread panicked while panicking. \
+                                       aborting."));
+        unsafe { intrinsics::abort() }
     }
 }
diff --git a/src/libstd/sys/common/unwind/mod.rs b/src/libstd/sys/common/unwind/mod.rs
index 8148bb6b7b8..c06d7886a75 100644
--- a/src/libstd/sys/common/unwind/mod.rs
+++ b/src/libstd/sys/common/unwind/mod.rs
@@ -64,9 +64,8 @@ use prelude::v1::*;
 
 use any::Any;
 use boxed;
-use cell::Cell;
 use cmp;
-use panicking;
+use panicking::{self,PANIC_COUNT};
 use fmt;
 use intrinsics;
 use mem;
@@ -92,8 +91,6 @@ pub mod imp;
 #[path = "gcc.rs"] #[doc(hidden)]
 pub mod imp;
 
-thread_local! { static PANICKING: Cell<bool> = Cell::new(false) }
-
 /// Invoke a closure, capturing the cause of panic if one occurs.
 ///
 /// This function will return `Ok(())` if the closure did not panic, and will
@@ -131,9 +128,9 @@ pub unsafe fn try<F: FnOnce()>(f: F) -> Result<(), Box<Any + Send>> {
     // care of exposing correctly.
     unsafe fn inner_try(f: fn(*mut u8), data: *mut u8)
                         -> Result<(), Box<Any + Send>> {
-        PANICKING.with(|s| {
+        PANIC_COUNT.with(|s| {
             let prev = s.get();
-            s.set(false);
+            s.set(0);
             let ep = intrinsics::try(f, data);
             s.set(prev);
             if ep.is_null() {
@@ -161,7 +158,7 @@ pub unsafe fn try<F: FnOnce()>(f: F) -> Result<(), Box<Any + Send>> {
 
 /// Determines whether the current thread is unwinding because of panic.
 pub fn panicking() -> bool {
-    PANICKING.with(|s| s.get())
+    PANIC_COUNT.with(|s| s.get() != 0)
 }
 
 // An uninlined, unmangled function upon which to slap yer breakpoints
@@ -234,17 +231,6 @@ fn begin_unwind_inner(msg: Box<Any + Send>,
     // First, invoke the default panic handler.
     panicking::on_panic(&*msg, file, line);
 
-    if panicking() {
-        // If a thread panics while it's already unwinding then we
-        // have limited options. Currently our preference is to
-        // just abort. In the future we may consider resuming
-        // unwinding or otherwise exiting the thread cleanly.
-        super::util::dumb_print(format_args!("thread panicked while panicking. \
-                                              aborting."));
-        unsafe { intrinsics::abort() }
-    }
-    PANICKING.with(|s| s.set(true));
-
     // Finally, perform the unwinding.
     rust_panic(msg);
 }