Auto merge of #65646 - Amanieu:foreign-exceptions, r=nikomatsakis

Allow foreign exceptions to unwind through Rust code and Rust panics to unwind through FFI

This PR fixes interactions between Rust panics and foreign (mainly C++) exceptions.

C++ exceptions (and other FFI exceptions) can now safely unwind through Rust code:
- The FFI function causing the unwind must be marked with `#[unwind(allowed)]`. If this is not the case then LLVM may optimize landing pads away with the assumption that they are unreachable.
- Drop code will be executed as the exception unwinds through the stack, as with a Rust panic.
- `catch_unwind` will *not* catch the exception, instead the exception will silently continue unwinding past it.

Rust panics can now safely unwind through C++ code:
- C++ destructors will be called as the stack unwinds.
- The Rust panic can only be caught with `catch (...)`, after which it can be either rethrown or discarded.
- C++ cannot name the type of the Rust exception object used for unwinding, which means that it can't be caught explicitly or have its contents inspected.

Tests have been added to ensure all of the above works correctly.

Some notes about non-C++ exceptions:
- `pthread_cancel` and `pthread_exit` use unwinding on glibc. This has the same behavior as a C++ exception: destructors are run but it cannot be caught by `catch_unwind`.
- `longjmp` on Windows is implemented using unwinding. Destructors are run on MSVC, but not on MinGW. In both cases the unwind cannot be caught by `catch_unwind`.
- As with C++ exceptions, you need to mark the relevant FFI functions with `#[unwind(allowed)]`, otherwise LLVM will optimize out the destructors since they seem unreachable.

I haven't updated any of the documentation, so officially unwinding through FFI is still UB. However this is a step towards making it well-defined.

Fixes #65441

cc @gnzlbg
r? @alexcrichton

6 files changed, 161 insertions, 2 deletions

diff --git a/src/test/run-make-fulldeps/foreign-exceptions/Makefile b/src/test/run-make-fulldeps/foreign-exceptions/Makefile
new file mode 100644
index 00000000000..7eba52f3c24
--- /dev/null
+++ b/src/test/run-make-fulldeps/foreign-exceptions/Makefile
@@ -0,0 +1,10 @@
+-include ../tools.mk
+
+all: foo
+	$(call RUN,foo)
+
+foo: foo.rs $(call NATIVE_STATICLIB,foo)
+	$(RUSTC) $< -lfoo $(EXTRARSCXXFLAGS)
+
+$(TMPDIR)/libfoo.o: foo.cpp
+	$(call COMPILE_OBJ_CXX,$@,$<)
diff --git a/src/test/run-make-fulldeps/foreign-exceptions/foo.cpp b/src/test/run-make-fulldeps/foreign-exceptions/foo.cpp
new file mode 100644
index 00000000000..b0fd65f88e7
--- /dev/null
+++ b/src/test/run-make-fulldeps/foreign-exceptions/foo.cpp
@@ -0,0 +1,60 @@
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+
+void println(const char* s) {
+    puts(s);
+    fflush(stdout);
+}
+
+struct exception {};
+struct rust_panic {};
+
+struct drop_check {
+    bool* ok;
+    ~drop_check() {
+        println("~drop_check");
+
+        if (ok)
+            *ok = true;
+    }
+};
+
+extern "C" {
+    void rust_catch_callback(void (*cb)(), bool* rust_ok);
+
+    static void callback() {
+        println("throwing C++ exception");
+        throw exception();
+    }
+
+    void throw_cxx_exception() {
+        bool rust_ok = false;
+        try {
+            rust_catch_callback(callback, &rust_ok);
+            assert(false && "unreachable");
+        } catch (exception e) {
+            println("caught C++ exception");
+            assert(rust_ok);
+            return;
+        }
+        assert(false && "did not catch thrown C++ exception");
+    }
+
+    void cxx_catch_callback(void (*cb)(), bool* cxx_ok) {
+        drop_check x;
+        x.ok = NULL;
+        try {
+            cb();
+        } catch (rust_panic e) {
+            assert(false && "shouldn't be able to catch a rust panic");
+        } catch (...) {
+            println("caught foreign exception in catch (...)");
+            // Foreign exceptions are caught by catch (...). We only set the ok
+            // flag if we successfully caught the panic. The destructor of
+            // drop_check will then set the flag to true if it is executed.
+            x.ok = cxx_ok;
+            throw;
+        }
+    }
+}
diff --git a/src/test/run-make-fulldeps/foreign-exceptions/foo.rs b/src/test/run-make-fulldeps/foreign-exceptions/foo.rs
new file mode 100644
index 00000000000..399c78f8d2d
--- /dev/null
+++ b/src/test/run-make-fulldeps/foreign-exceptions/foo.rs
@@ -0,0 +1,66 @@
+// Tests that C++ exceptions can unwind through Rust code, run destructors and
+// are ignored by catch_unwind. Also tests that Rust panics can unwind through
+// C++ code.
+
+// For linking libstdc++ on MinGW
+#![cfg_attr(all(windows, target_env = "gnu"), feature(static_nobundle))]
+
+#![feature(unwind_attributes)]
+
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
+struct DropCheck<'a>(&'a mut bool);
+impl<'a> Drop for DropCheck<'a> {
+    fn drop(&mut self) {
+        println!("DropCheck::drop");
+        *self.0 = true;
+    }
+}
+
+extern "C" {
+    fn throw_cxx_exception();
+
+    #[unwind(allowed)]
+    fn cxx_catch_callback(cb: extern "C" fn(), ok: *mut bool);
+}
+
+#[no_mangle]
+#[unwind(allowed)]
+extern "C" fn rust_catch_callback(cb: extern "C" fn(), rust_ok: &mut bool) {
+    let _caught_unwind = catch_unwind(AssertUnwindSafe(|| {
+        let _drop = DropCheck(rust_ok);
+        cb();
+        unreachable!("should have unwound instead of returned");
+    }));
+    unreachable!("catch_unwind should not have caught foreign exception");
+}
+
+fn throw_rust_panic() {
+    #[unwind(allowed)]
+    extern "C" fn callback() {
+        println!("throwing rust panic");
+        panic!(1234i32);
+    }
+
+    let mut dropped = false;
+    let mut cxx_ok = false;
+    let caught_unwind = catch_unwind(AssertUnwindSafe(|| {
+        let _drop = DropCheck(&mut dropped);
+        unsafe {
+            cxx_catch_callback(callback, &mut cxx_ok);
+        }
+        unreachable!("should have unwound instead of returned");
+    }));
+    println!("caught rust panic");
+    assert!(dropped);
+    assert!(caught_unwind.is_err());
+    let panic_obj = caught_unwind.unwrap_err();
+    let panic_int = *panic_obj.downcast_ref::<i32>().unwrap();
+    assert_eq!(panic_int, 1234);
+    assert!(cxx_ok);
+}
+
+fn main() {
+    unsafe { throw_cxx_exception() };
+    throw_rust_panic();
+}
diff --git a/src/test/run-make-fulldeps/issue-36710/Makefile b/src/test/run-make-fulldeps/issue-36710/Makefile
index dc1fbb4cefb..4f93d97636e 100644
--- a/src/test/run-make-fulldeps/issue-36710/Makefile
+++ b/src/test/run-make-fulldeps/issue-36710/Makefile
@@ -6,7 +6,7 @@ all: foo
 	$(call RUN,foo)
 
 foo: foo.rs $(call NATIVE_STATICLIB,foo)
-	$(RUSTC) $< -lfoo $(EXTRACXXFLAGS)
+	$(RUSTC) $< -lfoo $(EXTRARSCXXFLAGS)
 
 $(TMPDIR)/libfoo.o: foo.cpp
 	$(call COMPILE_OBJ_CXX,$@,$<)
diff --git a/src/test/run-make-fulldeps/issue-36710/foo.rs b/src/test/run-make-fulldeps/issue-36710/foo.rs
index a9d3effbd18..061f07c3243 100644
--- a/src/test/run-make-fulldeps/issue-36710/foo.rs
+++ b/src/test/run-make-fulldeps/issue-36710/foo.rs
@@ -1,5 +1,8 @@
 // Tests that linking to C++ code with global destructors works.
 
+// For linking libstdc++ on MinGW
+#![cfg_attr(all(windows, target_env = "gnu"), feature(static_nobundle))]
+
 extern { fn get() -> u32; }
 
 fn main() {
diff --git a/src/test/run-make-fulldeps/tools.mk b/src/test/run-make-fulldeps/tools.mk
index 9a113b7fa63..20a5e8e6422 100644
--- a/src/test/run-make-fulldeps/tools.mk
+++ b/src/test/run-make-fulldeps/tools.mk
@@ -60,7 +60,7 @@ endif
 
 ifdef IS_MSVC
 COMPILE_OBJ = $(CC) -c -Fo:`cygpath -w $(1)` $(2)
-COMPILE_OBJ_CXX = $(CXX) -c -Fo:`cygpath -w $(1)` $(2)
+COMPILE_OBJ_CXX = $(CXX) -EHs -c -Fo:`cygpath -w $(1)` $(2)
 NATIVE_STATICLIB_FILE = $(1).lib
 NATIVE_STATICLIB = $(TMPDIR)/$(call NATIVE_STATICLIB_FILE,$(1))
 OUT_EXE=-Fe:`cygpath -w $(TMPDIR)/$(call BIN,$(1))` \
@@ -80,10 +80,29 @@ ifdef IS_MSVC
 	EXTRACFLAGS := ws2_32.lib userenv.lib advapi32.lib
 else
 	EXTRACFLAGS := -lws2_32 -luserenv
+	EXTRACXXFLAGS := -lstdc++
+	# So this is a bit hacky: we can't use the DLL version of libstdc++ because
+	# it pulls in the DLL version of libgcc, which means that we end up with 2
+	# instances of the DW2 unwinding implementation. This is a problem on
+	# i686-pc-windows-gnu because each module (DLL/EXE) needs to register its
+	# unwind information with the unwinding implementation, and libstdc++'s
+	# __cxa_throw won't see the unwinding info we registered with our statically
+	# linked libgcc.
+	#
+	# Now, simply statically linking libstdc++ would fix this problem, except
+	# that it is compiled with the expectation that pthreads is dynamically
+	# linked as a DLL and will fail to link with a statically linked libpthread.
+	#
+	# So we end up with the following hack: we link use static-nobundle to only
+	# link the parts of libstdc++ that we actually use, which doesn't include
+	# the dependency on the pthreads DLL.
+	EXTRARSCXXFLAGS := -l static-nobundle=stdc++
 endif
 else
 ifeq ($(UNAME),Darwin)
 	EXTRACFLAGS := -lresolv
+	EXTRACXXFLAGS := -lc++
+	EXTRARSCXXFLAGS := -lc++
 else
 ifeq ($(UNAME),FreeBSD)
 	EXTRACFLAGS := -lm -lpthread -lgcc_s
@@ -97,6 +116,7 @@ ifeq ($(UNAME),OpenBSD)
 else
 	EXTRACFLAGS := -lm -lrt -ldl -lpthread
 	EXTRACXXFLAGS := -lstdc++
+	EXTRARSCXXFLAGS := -lstdc++
 endif
 endif
 endif