Rollup merge of #30801 - Amanieu:oom_print, r=alexcrichton

This adds the ability to override the default OOM behavior by setting a handler function. This is used by libstd to print a message when running out of memory instead of crashing with an obscure "illegal hardware instruction" error (at least on Linux).

Fixes #14674

3 files changed, 45 insertions, 4 deletions

diff --git a/src/libstd/sys/common/util.rs b/src/libstd/sys/common/util.rs
index 979f1f48669..b7a6b7650d5 100644
--- a/src/libstd/sys/common/util.rs
+++ b/src/libstd/sys/common/util.rs
@@ -35,12 +35,12 @@ pub fn dumb_print(args: fmt::Arguments) {
 }
 
 pub fn abort(args: fmt::Arguments) -> ! {
-    dumb_print(format_args!("fatal runtime error: {}", args));
+    dumb_print(format_args!("fatal runtime error: {}\n", args));
     unsafe { intrinsics::abort(); }
 }
 
 #[allow(dead_code)] // stack overflow detection not enabled on all platforms
 pub unsafe fn report_overflow() {
-    dumb_print(format_args!("\nthread '{}' has overflowed its stack",
+    dumb_print(format_args!("\nthread '{}' has overflowed its stack\n",
                             thread::current().name().unwrap_or("<unknown>")));
 }
diff --git a/src/libstd/sys/unix/mod.rs b/src/libstd/sys/unix/mod.rs
index 929fd2fb0c3..9771b057d8d 100644
--- a/src/libstd/sys/unix/mod.rs
+++ b/src/libstd/sys/unix/mod.rs
@@ -15,6 +15,7 @@ use io::{self, ErrorKind};
 use libc;
 use num::One;
 use ops::Neg;
+use alloc::oom;
 
 #[cfg(target_os = "android")]   pub use os::android as platform;
 #[cfg(target_os = "bitrig")]    pub use os::bitrig as platform;
@@ -45,6 +46,22 @@ pub mod thread_local;
 pub mod time;
 pub mod stdio;
 
+// A nicer handler for out-of-memory situations than the default one. This one
+// prints a message to stderr before aborting. It is critical that this code
+// does not allocate any memory since we are in an OOM situation. Any errors are
+// ignored while printing since there's nothing we can do about them and we are
+// about to exit anyways.
+fn oom_handler() -> ! {
+    use intrinsics;
+    let msg = "fatal runtime error: out of memory\n";
+    unsafe {
+        libc::write(libc::STDERR_FILENO,
+                    msg.as_ptr() as *const libc::c_void,
+                    msg.len() as libc::size_t);
+        intrinsics::abort();
+    }
+}
+
 #[cfg(not(any(target_os = "nacl", test)))]
 pub fn init() {
     use libc::signal;
@@ -58,10 +75,14 @@ pub fn init() {
     unsafe {
         assert!(signal(libc::SIGPIPE, libc::SIG_IGN) != !0);
     }
+
+    oom::set_oom_handler(oom_handler);
 }
 
 #[cfg(all(target_os = "nacl", not(test)))]
-pub fn init() { }
+pub fn init() {
+    oom::set_oom_handler(oom_handler);
+}
 
 pub fn decode_error_kind(errno: i32) -> ErrorKind {
     match errno as libc::c_int {
diff --git a/src/libstd/sys/windows/mod.rs b/src/libstd/sys/windows/mod.rs
index 7e5342a3fd4..16c4ae8257c 100644
--- a/src/libstd/sys/windows/mod.rs
+++ b/src/libstd/sys/windows/mod.rs
@@ -20,6 +20,7 @@ use num::Zero;
 use os::windows::ffi::{OsStrExt, OsStringExt};
 use path::PathBuf;
 use time::Duration;
+use alloc::oom;
 
 #[macro_use] pub mod compat;
 
@@ -42,7 +43,26 @@ pub mod thread_local;
 pub mod time;
 pub mod stdio;
 
-pub fn init() {}
+// See comment in sys/unix/mod.rs
+fn oom_handler() -> ! {
+    use intrinsics;
+    use ptr;
+    let msg = "fatal runtime error: out of memory\n";
+    unsafe {
+        // WriteFile silently fails if it is passed an invalid handle, so there
+        // is no need to check the result of GetStdHandle.
+        c::WriteFile(c::GetStdHandle(c::STD_ERROR_HANDLE),
+                     msg.as_ptr() as c::LPVOID,
+                     msg.len() as c::DWORD,
+                     ptr::null_mut(),
+                     ptr::null_mut());
+        intrinsics::abort();
+    }
+}
+
+pub fn init() {
+    oom::set_oom_handler(oom_handler);
+}
 
 pub fn decode_error_kind(errno: i32) -> ErrorKind {
     match errno as c::DWORD {