auto merge of #14006 : thestinger/rust/jemalloc, r=alexcrichton

Closes #11807

6 files changed, 303 insertions, 149 deletions

diff --git a/src/libstd/rt/global_heap.rs b/src/libstd/rt/global_heap.rs
deleted file mode 100644
index 7d54c3faf42..00000000000
--- a/src/libstd/rt/global_heap.rs
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-
-//! The global (exchange) heap.
-
-use libc::{c_void, size_t, free, malloc, realloc};
-use ptr::{RawPtr, mut_null};
-use intrinsics::abort;
-use raw;
-use mem::size_of;
-
-#[inline]
-pub fn get_box_size(body_size: uint, body_align: uint) -> uint {
-    let header_size = size_of::<raw::Box<()>>();
-    let total_size = align_to(header_size, body_align) + body_size;
-    total_size
-}
-
-// Rounds |size| to the nearest |alignment|. Invariant: |alignment| is a power
-// of two.
-#[inline]
-fn align_to(size: uint, align: uint) -> uint {
-    assert!(align != 0);
-    (size + align - 1) & !(align - 1)
-}
-
-/// A wrapper around libc::malloc, aborting on out-of-memory
-#[inline]
-pub unsafe fn malloc_raw(size: uint) -> *mut u8 {
-    // `malloc(0)` may allocate, but it may also return a null pointer
-    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/malloc.html
-    if size == 0 {
-        mut_null()
-    } else {
-        let p = malloc(size as size_t);
-        if p.is_null() {
-            // we need a non-allocating way to print an error here
-            abort();
-        }
-        p as *mut u8
-    }
-}
-
-/// A wrapper around libc::realloc, aborting on out-of-memory
-#[inline]
-pub unsafe fn realloc_raw(ptr: *mut u8, size: uint) -> *mut u8 {
-    // `realloc(ptr, 0)` may allocate, but it may also return a null pointer
-    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/realloc.html
-    if size == 0 {
-        free(ptr as *mut c_void);
-        mut_null()
-    } else {
-        let p = realloc(ptr as *mut c_void, size as size_t);
-        if p.is_null() {
-            // we need a non-allocating way to print an error here
-            abort();
-        }
-        p as *mut u8
-    }
-}
-
-/// The allocator for unique pointers without contained managed pointers.
-#[cfg(not(test))]
-#[lang="exchange_malloc"]
-#[inline]
-pub unsafe fn exchange_malloc(size: uint) -> *mut u8 {
-    // The compiler never calls `exchange_free` on Box<ZeroSizeType>, so
-    // zero-size allocations can point to this `static`. It would be incorrect
-    // to use a null pointer, due to enums assuming types like unique pointers
-    // are never null.
-    static EMPTY: () = ();
-
-    if size == 0 {
-        &EMPTY as *() as *mut u8
-    } else {
-        malloc_raw(size)
-    }
-}
-
-// FIXME: #7496
-#[cfg(not(test))]
-#[lang="closure_exchange_malloc"]
-#[inline]
-pub unsafe fn closure_exchange_malloc_(drop_glue: fn(*mut u8), size: uint, align: uint) -> *u8 {
-    closure_exchange_malloc(drop_glue, size, align)
-}
-
-#[inline]
-pub unsafe fn closure_exchange_malloc(drop_glue: fn(*mut u8), size: uint, align: uint) -> *u8 {
-    let total_size = get_box_size(size, align);
-    let p = malloc_raw(total_size);
-
-    let alloc = p as *mut raw::Box<()>;
-    (*alloc).drop_glue = drop_glue;
-
-    alloc as *u8
-}
-
-// NB: Calls to free CANNOT be allowed to fail, as throwing an exception from
-// inside a landing pad may corrupt the state of the exception handler.
-#[cfg(not(test))]
-#[lang="exchange_free"]
-#[inline]
-pub unsafe fn exchange_free_(ptr: *u8) {
-    exchange_free(ptr)
-}
-
-#[inline]
-pub unsafe fn exchange_free(ptr: *u8) {
-    free(ptr as *mut c_void);
-}
-
-#[cfg(test)]
-mod bench {
-    extern crate test;
-    use self::test::Bencher;
-
-    #[bench]
-    fn alloc_owned_small(b: &mut Bencher) {
-        b.iter(|| {
-            box 10
-        })
-    }
-
-    #[bench]
-    fn alloc_owned_big(b: &mut Bencher) {
-        b.iter(|| {
-            box [10, ..1000]
-        })
-    }
-}
diff --git a/src/libstd/rt/heap.rs b/src/libstd/rt/heap.rs
new file mode 100644
index 00000000000..ffe6dccfa15
--- /dev/null
+++ b/src/libstd/rt/heap.rs
@@ -0,0 +1,222 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// FIXME: #13994: port to the sized deallocation API when available
+// FIXME: #13996: need a way to mark the `allocate` and `reallocate` return values as `noalias`
+
+use intrinsics::{abort, cttz32};
+use libc::{c_int, c_void, size_t};
+use ptr::RawPtr;
+
+#[link(name = "jemalloc", kind = "static")]
+extern {
+    fn je_mallocx(size: size_t, flags: c_int) -> *mut c_void;
+    fn je_rallocx(ptr: *mut c_void, size: size_t, flags: c_int) -> *mut c_void;
+    fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t, flags: c_int) -> size_t;
+    fn je_dallocx(ptr: *mut c_void, flags: c_int);
+    fn je_nallocx(size: size_t, flags: c_int) -> size_t;
+}
+
+// -lpthread needs to occur after -ljemalloc, the earlier argument isn't enough
+#[cfg(not(windows), not(target_os = "android"))]
+#[link(name = "pthread")]
+extern {}
+
+// MALLOCX_ALIGN(a) macro
+#[inline(always)]
+fn mallocx_align(a: uint) -> c_int { unsafe { cttz32(a as u32) as c_int } }
+
+/// Return a pointer to `size` bytes of memory.
+///
+/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The
+/// alignment must be no larger than the largest supported page size on the platform.
+#[inline]
+pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
+    let ptr = je_mallocx(size as size_t, mallocx_align(align)) as *mut u8;
+    if ptr.is_null() {
+        abort()
+    }
+    ptr
+}
+
+/// Extend or shrink the allocation referenced by `ptr` to `size` bytes of memory.
+///
+/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The
+/// alignment must be no larger than the largest supported page size on the platform.
+///
+/// The `old_size` and `align` parameters are the parameters that were used to create the
+/// allocation referenced by `ptr`. The `old_size` parameter may also be the value returned by
+/// `usable_size` for the requested size.
+#[inline]
+#[allow(unused_variable)] // for the parameter names in the documentation
+pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint, old_size: uint) -> *mut u8 {
+    let ptr = je_rallocx(ptr as *mut c_void, size as size_t, mallocx_align(align)) as *mut u8;
+    if ptr.is_null() {
+        abort()
+    }
+    ptr
+}
+
+/// Extend or shrink the allocation referenced by `ptr` to `size` bytes of memory in-place.
+///
+/// Return true if successful, otherwise false if the allocation was not altered.
+///
+/// Behavior is undefined if the requested size is 0 or the alignment is not a power of 2. The
+/// alignment must be no larger than the largest supported page size on the platform.
+///
+/// The `old_size` and `align` parameters are the parameters that were used to
+/// create the allocation referenced by `ptr`. The `old_size` parameter may be
+/// any value in range_inclusive(requested_size, usable_size).
+#[inline]
+#[allow(unused_variable)] // for the parameter names in the documentation
+pub unsafe fn reallocate_inplace(ptr: *mut u8, size: uint, align: uint, old_size: uint) -> bool {
+    je_xallocx(ptr as *mut c_void, size as size_t, 0, mallocx_align(align)) == size as size_t
+}
+
+/// Deallocate the memory referenced by `ptr`.
+///
+/// The `ptr` parameter must not be null.
+///
+/// The `size` and `align` parameters are the parameters that were used to create the
+/// allocation referenced by `ptr`. The `size` parameter may also be the value returned by
+/// `usable_size` for the requested size.
+#[inline]
+#[allow(unused_variable)] // for the parameter names in the documentation
+pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
+    je_dallocx(ptr as *mut c_void, mallocx_align(align))
+}
+
+/// Return the usable size of an allocation created with the specified the `size` and `align`.
+#[inline]
+pub fn usable_size(size: uint, align: uint) -> uint {
+    unsafe { je_nallocx(size as size_t, mallocx_align(align)) as uint }
+}
+
+/// The allocator for unique pointers.
+#[cfg(stage0)]
+#[lang="exchange_malloc"]
+#[inline(always)]
+pub unsafe fn exchange_malloc_(size: uint) -> *mut u8 {
+    exchange_malloc(size)
+}
+
+/// The allocator for unique pointers.
+#[cfg(not(test), not(stage0))]
+#[lang="exchange_malloc"]
+#[inline(always)]
+pub unsafe fn exchange_malloc_(size: uint, align: uint) -> *mut u8 {
+    exchange_malloc(size, align)
+}
+
+/// The allocator for unique pointers.
+#[cfg(stage0)]
+#[inline]
+pub unsafe fn exchange_malloc(size: uint) -> *mut u8 {
+    // The compiler never calls `exchange_free` on ~ZeroSizeType, so zero-size
+    // allocations can point to this `static`. It would be incorrect to use a null
+    // pointer, due to enums assuming types like unique pointers are never null.
+    static EMPTY: () = ();
+
+    if size == 0 {
+        &EMPTY as *() as *mut u8
+    } else {
+        allocate(size, 8)
+    }
+}
+
+/// The allocator for unique pointers.
+#[cfg(not(stage0))]
+#[inline]
+pub unsafe fn exchange_malloc(size: uint, align: uint) -> *mut u8 {
+    // The compiler never calls `exchange_free` on ~ZeroSizeType, so zero-size
+    // allocations can point to this `static`. It would be incorrect to use a null
+    // pointer, due to enums assuming types like unique pointers are never null.
+    static EMPTY: () = ();
+
+    if size == 0 {
+        &EMPTY as *() as *mut u8
+    } else {
+        allocate(size, align)
+    }
+}
+
+#[cfg(not(test))]
+#[lang="exchange_free"]
+#[inline]
+// FIXME: #13994 (rustc should pass align and size here)
+pub unsafe fn exchange_free_(ptr: *mut u8) {
+    exchange_free(ptr, 0, 8)
+}
+
+#[inline]
+pub unsafe fn exchange_free(ptr: *mut u8, size: uint, align: uint) {
+    deallocate(ptr, size, align);
+}
+
+// FIXME: #7496
+#[cfg(not(test))]
+#[lang="closure_exchange_malloc"]
+#[inline]
+unsafe fn closure_exchange_malloc(drop_glue: fn(*mut u8), size: uint, align: uint) -> *mut u8 {
+    let total_size = ::rt::util::get_box_size(size, align);
+    let p = allocate(total_size, 8);
+
+    let alloc = p as *mut ::raw::Box<()>;
+    (*alloc).drop_glue = drop_glue;
+
+    alloc as *mut u8
+}
+
+// hack for libcore
+#[no_mangle]
+#[doc(hidden)]
+#[deprecated]
+#[cfg(stage0, not(test))]
+pub extern "C" fn rust_malloc(size: uint) -> *mut u8 {
+    unsafe { exchange_malloc(size) }
+}
+
+// hack for libcore
+#[no_mangle]
+#[doc(hidden)]
+#[deprecated]
+#[cfg(not(stage0), not(test))]
+pub extern "C" fn rust_malloc(size: uint, align: uint) -> *mut u8 {
+    unsafe { exchange_malloc(size, align) }
+}
+
+// hack for libcore
+#[no_mangle]
+#[doc(hidden)]
+#[deprecated]
+#[cfg(not(test))]
+pub extern "C" fn rust_free(ptr: *mut u8, size: uint, align: uint) {
+    unsafe { exchange_free(ptr, size, align) }
+}
+
+#[cfg(test)]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+
+    #[bench]
+    fn alloc_owned_small(b: &mut Bencher) {
+        b.iter(|| {
+            box 10
+        })
+    }
+
+    #[bench]
+    fn alloc_owned_big(b: &mut Bencher) {
+        b.iter(|| {
+            box [10, ..1000]
+        })
+    }
+}
diff --git a/src/libstd/rt/libc_heap.rs b/src/libstd/rt/libc_heap.rs
new file mode 100644
index 00000000000..ece51ab9989
--- /dev/null
+++ b/src/libstd/rt/libc_heap.rs
@@ -0,0 +1,51 @@
+// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+
+//! The global (exchange) heap.
+
+use libc::{c_void, size_t, free, malloc, realloc};
+use ptr::{RawPtr, mut_null};
+use intrinsics::abort;
+
+/// A wrapper around libc::malloc, aborting on out-of-memory
+#[inline]
+pub unsafe fn malloc_raw(size: uint) -> *mut u8 {
+    // `malloc(0)` may allocate, but it may also return a null pointer
+    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/malloc.html
+    if size == 0 {
+        mut_null()
+    } else {
+        let p = malloc(size as size_t);
+        if p.is_null() {
+            // we need a non-allocating way to print an error here
+            abort();
+        }
+        p as *mut u8
+    }
+}
+
+/// A wrapper around libc::realloc, aborting on out-of-memory
+#[inline]
+pub unsafe fn realloc_raw(ptr: *mut u8, size: uint) -> *mut u8 {
+    // `realloc(ptr, 0)` may allocate, but it may also return a null pointer
+    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/realloc.html
+    if size == 0 {
+        free(ptr as *mut c_void);
+        mut_null()
+    } else {
+        let p = realloc(ptr as *mut c_void, size as size_t);
+        if p.is_null() {
+            // we need a non-allocating way to print an error here
+            abort();
+        }
+        p as *mut u8
+    }
+}
diff --git a/src/libstd/rt/local_heap.rs b/src/libstd/rt/local_heap.rs
index caf0d9028c5..efc8072594b 100644
--- a/src/libstd/rt/local_heap.rs
+++ b/src/libstd/rt/local_heap.rs
@@ -12,12 +12,13 @@
 
 use cast;
 use iter::Iterator;
+use libc::{c_void, free};
 use mem;
 use ops::Drop;
 use option::{Option, None, Some};
 use ptr;
 use ptr::RawPtr;
-use rt::global_heap;
+use rt::libc_heap;
 use rt::local::Local;
 use rt::task::Task;
 use raw;
@@ -58,7 +59,7 @@ impl LocalHeap {
 
     #[inline]
     pub fn alloc(&mut self, drop_glue: fn(*mut u8), size: uint, align: uint) -> *mut Box {
-        let total_size = global_heap::get_box_size(size, align);
+        let total_size = ::rt::util::get_box_size(size, align);
         let alloc = self.memory_region.malloc(total_size);
         {
             // Make sure that we can't use `mybox` outside of this scope
@@ -187,7 +188,7 @@ impl MemoryRegion {
     fn malloc(&mut self, size: uint) -> *mut Box {
         let total_size = size + AllocHeader::size();
         let alloc: *AllocHeader = unsafe {
-            global_heap::malloc_raw(total_size) as *AllocHeader
+            libc_heap::malloc_raw(total_size) as *AllocHeader
         };
 
         let alloc: &mut AllocHeader = unsafe { cast::transmute(alloc) };
@@ -206,8 +207,7 @@ impl MemoryRegion {
 
         let total_size = size + AllocHeader::size();
         let alloc: *AllocHeader = unsafe {
-            global_heap::realloc_raw(orig_alloc as *mut u8,
-                                     total_size) as *AllocHeader
+            libc_heap::realloc_raw(orig_alloc as *mut u8, total_size) as *AllocHeader
         };
 
         let alloc: &mut AllocHeader = unsafe { cast::transmute(alloc) };
@@ -226,7 +226,7 @@ impl MemoryRegion {
             self.release(cast::transmute(alloc));
             rtassert!(self.live_allocations > 0);
             self.live_allocations -= 1;
-            global_heap::exchange_free(alloc as *u8)
+            free(alloc as *mut c_void)
         }
     }
 
diff --git a/src/libstd/rt/mod.rs b/src/libstd/rt/mod.rs
index 5b9c314d42b..a04cbabedd6 100644
--- a/src/libstd/rt/mod.rs
+++ b/src/libstd/rt/mod.rs
@@ -26,7 +26,7 @@ language and an implementation must be provided regardless of the
 execution environment.
 
 Of foremost importance is the global exchange heap, in the module
-`global_heap`. Very little practical Rust code can be written without
+`heap`. Very little practical Rust code can be written without
 access to the global heap. Unlike most of `rt` the global heap is
 truly a global resource and generally operates independently of the
 rest of the runtime.
@@ -86,10 +86,13 @@ pub mod shouldnt_be_public {
 // Internal macros used by the runtime.
 mod macros;
 
-// The global (exchange) heap.
-pub mod global_heap;
+/// Wrappers around malloc / realloc aborting on out-of-memory.
+pub mod libc_heap;
 
-// Implementations of language-critical runtime features like @.
+/// The low-level memory allocation API.
+pub mod heap;
+
+/// Implementations of language-critical runtime features like @.
 pub mod task;
 
 // The EventLoop and internal synchronous I/O interface.
diff --git a/src/libstd/rt/util.rs b/src/libstd/rt/util.rs
index 84284ca1faf..e8b1acb1024 100644
--- a/src/libstd/rt/util.rs
+++ b/src/libstd/rt/util.rs
@@ -26,6 +26,23 @@ use slice::ImmutableVector;
 // FIXME: Once the runtime matures remove the `true` below to turn off rtassert, etc.
 pub static ENFORCE_SANITY: bool = true || !cfg!(rtopt) || cfg!(rtdebug) || cfg!(rtassert);
 
+#[deprecated]
+#[doc(hidden)]
+#[inline]
+pub fn get_box_size(body_size: uint, body_align: uint) -> uint {
+    let header_size = ::mem::size_of::<::raw::Box<()>>();
+    let total_size = align_to(header_size, body_align) + body_size;
+    total_size
+}
+
+// Rounds |size| to the nearest |alignment|. Invariant: |alignment| is a power
+// of two.
+#[inline]
+fn align_to(size: uint, align: uint) -> uint {
+    assert!(align != 0);
+    (size + align - 1) & !(align - 1)
+}
+
 /// Get the number of cores available
 pub fn num_cpus() -> uint {
     unsafe {