handle zero-size allocations correctly

The `malloc` family of functions may return a null pointer for a
zero-size allocation, which should not be interpreted as an
out-of-memory error.

If the implementation does not return a null pointer, then handling
this will result in memory savings for zero-size types.

This also switches some code to `malloc_raw` in order to maintain a
centralized point for handling out-of-memory in `rt::global_heap`.

Closes #11634

3 files changed, 31 insertions, 21 deletions

diff --git a/src/libstd/libc.rs b/src/libstd/libc.rs
index 9cf94e5a1b8..77ac226a7f1 100644
--- a/src/libstd/libc.rs
+++ b/src/libstd/libc.rs
@@ -3223,7 +3223,7 @@ pub mod funcs {
                 pub fn strtoul(s: *c_char, endp: **c_char, base: c_int)
                                -> c_ulong;
                 pub fn calloc(nobj: size_t, size: size_t) -> *c_void;
-                pub fn malloc(size: size_t) -> *c_void;
+                pub fn malloc(size: size_t) -> *mut c_void;
                 pub fn realloc(p: *mut c_void, size: size_t) -> *mut c_void;
                 pub fn free(p: *c_void);
                 pub fn exit(status: c_int) -> !;
diff --git a/src/libstd/rt/global_heap.rs b/src/libstd/rt/global_heap.rs
index ce4072fb1ab..00195f726cb 100644
--- a/src/libstd/rt/global_heap.rs
+++ b/src/libstd/rt/global_heap.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use libc::{c_void, c_char, size_t, uintptr_t, free, malloc, realloc};
-use ptr::RawPtr;
+use ptr::{RawPtr, mut_null};
 use unstable::intrinsics::{TyDesc, abort};
 use unstable::raw;
 use mem::size_of;
@@ -31,24 +31,37 @@ fn align_to(size: uint, align: uint) -> uint {
 
 /// A wrapper around libc::malloc, aborting on out-of-memory
 #[inline]
-pub unsafe fn malloc_raw(size: uint) -> *c_void {
-    let p = malloc(size as size_t);
-    if p.is_null() {
-        // we need a non-allocating way to print an error here
-        abort();
+pub unsafe fn malloc_raw(size: uint) -> *mut c_void {
+    // `malloc(0)` may allocate, but it may also return a null pointer
+    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/malloc.html
+    if size == 0 {
+        mut_null()
+    } else {
+        let p = malloc(size as size_t);
+        if p.is_null() {
+            // we need a non-allocating way to print an error here
+            abort();
+        }
+        p
     }
-    p
 }
 
 /// A wrapper around libc::realloc, aborting on out-of-memory
 #[inline]
 pub unsafe fn realloc_raw(ptr: *mut c_void, size: uint) -> *mut c_void {
-    let p = realloc(ptr, size as size_t);
-    if p.is_null() {
-        // we need a non-allocating way to print an error here
-        abort();
+    // `realloc(ptr, 0)` may allocate, but it may also return a null pointer
+    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/realloc.html
+    if size == 0 {
+        free(ptr as *c_void);
+        mut_null()
+    } else {
+        let p = realloc(ptr, size as size_t);
+        if p.is_null() {
+            // we need a non-allocating way to print an error here
+            abort();
+        }
+        p
     }
-    p
 }
 
 /// The allocator for unique pointers without contained managed pointers.
diff --git a/src/libstd/unstable/mutex.rs b/src/libstd/unstable/mutex.rs
index 4d12435e01a..81317b7de79 100644
--- a/src/libstd/unstable/mutex.rs
+++ b/src/libstd/unstable/mutex.rs
@@ -167,7 +167,7 @@ mod imp {
     use libc::c_void;
     use libc;
     use ptr;
-    use ptr::RawPtr;
+    use rt::global_heap::malloc_raw;
 
     type pthread_mutex_t = libc::c_void;
     type pthread_mutexattr_t = libc::c_void;
@@ -175,16 +175,14 @@ mod imp {
     type pthread_condattr_t = libc::c_void;
 
     pub unsafe fn init_lock() -> uint {
-        let block = libc::malloc(rust_pthread_mutex_t_size() as libc::size_t);
-        assert!(!block.is_null());
+        let block = malloc_raw(rust_pthread_mutex_t_size() as uint) as *c_void;
         let n = pthread_mutex_init(block, ptr::null());
         assert_eq!(n, 0);
         return block as uint;
     }
 
     pub unsafe fn init_cond() -> uint {
-        let block = libc::malloc(rust_pthread_cond_t_size() as libc::size_t);
-        assert!(!block.is_null());
+        let block = malloc_raw(rust_pthread_cond_t_size() as uint) as *c_void;
         let n = pthread_cond_init(block, ptr::null());
         assert_eq!(n, 0);
         return block as uint;
@@ -249,14 +247,13 @@ mod imp {
     use libc;
     use libc::{HANDLE, BOOL, LPSECURITY_ATTRIBUTES, c_void, DWORD, LPCSTR};
     use ptr;
-    use ptr::RawPtr;
+    use rt::global_heap::malloc_raw;
 
     type LPCRITICAL_SECTION = *c_void;
     static SPIN_COUNT: DWORD = 4000;
 
     pub unsafe fn init_lock() -> uint {
-        let block = libc::malloc(rust_crit_section_size() as libc::size_t);
-        assert!(!block.is_null());
+        let block = malloc_raw(rust_crit_section_size() as uint) as *c_void;
         InitializeCriticalSectionAndSpinCount(block, SPIN_COUNT);
         return block as uint;
     }