diff options
Diffstat (limited to 'src/liballoc_jemalloc/lib.rs')
| -rw-r--r-- | src/liballoc_jemalloc/lib.rs | 49 |
1 files changed, 25 insertions, 24 deletions
diff --git a/src/liballoc_jemalloc/lib.rs b/src/liballoc_jemalloc/lib.rs index d153f19c462..d7370ae400d 100644 --- a/src/liballoc_jemalloc/lib.rs +++ b/src/liballoc_jemalloc/lib.rs @@ -72,8 +72,7 @@ mod contents { const MALLOCX_ZERO: c_int = 0x40; // The minimum alignment guaranteed by the architecture. This value is used to - // add fast paths for low alignment values. In practice, the alignment is a - // constant at the call site and the branch will be optimized out. + // add fast paths for low alignment values. #[cfg(all(any(target_arch = "arm", target_arch = "mips", target_arch = "powerpc")))] @@ -92,8 +91,8 @@ mod contents { a.trailing_zeros() as c_int } - fn align_to_flags(align: usize) -> c_int { - if align <= MIN_ALIGN { + fn align_to_flags(align: usize, size: usize) -> c_int { + if align <= MIN_ALIGN && align <= size { 0 } else { mallocx_align(align) @@ -107,11 +106,11 @@ mod contents { // ABI #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_alloc(size: usize, align: usize, err: *mut u8) -> *mut u8 { - let flags = align_to_flags(align); + let flags = align_to_flags(align, size); let ptr = mallocx(size as size_t, flags) as *mut u8; if ptr.is_null() { let layout = Layout::from_size_align_unchecked(size, align); @@ -122,27 +121,27 @@ mod contents { } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_oom(err: *const u8) -> ! { System.oom((*(err as *const AllocErr)).clone()) } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_dealloc(ptr: *mut u8, size: usize, align: usize) { - let flags = align_to_flags(align); + let flags = align_to_flags(align, size); sdallocx(ptr as *mut c_void, size, flags); } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_usable_size(layout: *const u8, min: *mut usize, max: *mut usize) { let layout = &*(layout as *const Layout); - let flags = align_to_flags(layout.align()); + let flags = align_to_flags(layout.align(), layout.size()); let size = nallocx(layout.size(), flags) as usize; *min = layout.size(); if size > 0 { @@ -153,7 +152,7 @@ mod contents { } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_realloc(ptr: *mut u8, _old_size: usize, old_align: usize, @@ -166,7 +165,7 @@ mod contents { return 0 as *mut u8 } - let flags = align_to_flags(new_align); + let flags = align_to_flags(new_align, new_size); let ptr = rallocx(ptr as *mut c_void, new_size, flags) as *mut u8; if ptr.is_null() { let layout = Layout::from_size_align_unchecked(new_size, new_align); @@ -177,14 +176,14 @@ mod contents { } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_alloc_zeroed(size: usize, align: usize, err: *mut u8) -> *mut u8 { - let ptr = if align <= MIN_ALIGN { + let ptr = if align <= MIN_ALIGN && align <= size { calloc(size as size_t, 1) as *mut u8 } else { - let flags = align_to_flags(align) | MALLOCX_ZERO; + let flags = align_to_flags(align, size) | MALLOCX_ZERO; mallocx(size as size_t, flags) as *mut u8 }; if ptr.is_null() { @@ -196,20 +195,21 @@ mod contents { } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_alloc_excess(size: usize, align: usize, excess: *mut usize, err: *mut u8) -> *mut u8 { let p = __rde_alloc(size, align, err); if !p.is_null() { - *excess = size; + let flags = align_to_flags(align, size); + *excess = nallocx(size, flags) as usize; } return p } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_realloc_excess(ptr: *mut u8, old_size: usize, old_align: usize, @@ -219,13 +219,14 @@ mod contents { err: *mut u8) -> *mut u8 { let p = __rde_realloc(ptr, old_size, old_align, new_size, new_align, err); if !p.is_null() { - *excess = new_size; + let flags = align_to_flags(new_align, new_size); + *excess = nallocx(new_size, flags) as usize; } - return p + p } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_grow_in_place(ptr: *mut u8, old_size: usize, old_align: usize, @@ -235,14 +236,14 @@ mod contents { } #[no_mangle] - #[linkage = "external"] + #[rustc_std_internal_symbol] pub unsafe extern fn __rde_shrink_in_place(ptr: *mut u8, _old_size: usize, old_align: usize, new_size: usize, new_align: usize) -> u8 { if old_align == new_align { - let flags = align_to_flags(new_align); + let flags = align_to_flags(new_align, new_size); (xallocx(ptr as *mut c_void, new_size, 0, flags) == new_size) as u8 } else { 0 |