about summary refs log tree commit diff
diff options
context:
space:
mode:
authorDylan DPC <99973273+Dylan-DPC@users.noreply.github.com>2022-10-26 11:29:53 +0530
committerGitHub <noreply@github.com>2022-10-26 11:29:53 +0530
commit8ed3a80b9a37ee31e69f1dbe2f720046f8d4422c (patch)
tree0c6778546b2c3b2fa2a29fa25ba5e837b307213c
parentf2c2e582bdb8ca9d1a09643d516e22b6d3406581 (diff)
parentcfcb0a2135da6d4bdbf5f28806dea0cc70c5b6c5 (diff)
downloadrust-8ed3a80b9a37ee31e69f1dbe2f720046f8d4422c.tar.gz
rust-8ed3a80b9a37ee31e69f1dbe2f720046f8d4422c.zip
Rollup merge of #103287 - saethlin:faster-len-check, r=thomcc
Use a faster allocation size check in slice::from_raw_parts

I've been perusing through the codegen changes that result from turning on the standard library debug assertions. The previous check in here uses saturating arithmetic, which in my experience sometimes makes LLVM just fail to optimize things around the saturating operation.

Here is a demo of the codegen difference: https://godbolt.org/z/WMEqrjajW
Before:
```asm
example::len_check_old:
        mov     rax, rdi
        mov     ecx, 3
        mul     rcx
        setno   cl
        test    rax, rax
        setns   al
        and     al, cl
        ret

example::len_check_old:
        mov     rax, rdi
        mov     ecx, 8
        mul     rcx
        setno   cl
        test    rax, rax
        setns   al
        and     al, cl
        ret
```
After:
```asm
example::len_check_new:
        movabs  rax, 3074457345618258603
        cmp     rdi, rax
        setb    al
        ret

example::len_check_new:
        shr     rdi, 60
        sete    al
        ret
```

Running rustc-perf locally, this looks like up to a 4.5% improvement when `debug-assertions-std = true`.

Thanks ```@LegionMammal978``` (I think that's you?) for turning my idea into a much cleaner implementation.

r? ```@thomcc```
-rw-r--r--library/core/src/intrinsics.rs10
-rw-r--r--library/core/src/lib.rs1
-rw-r--r--library/core/src/slice/raw.rs10
3 files changed, 16 insertions, 5 deletions
diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs
index 1589fee46be..29f796fad6d 100644
--- a/library/core/src/intrinsics.rs
+++ b/library/core/src/intrinsics.rs
@@ -2229,6 +2229,16 @@ pub(crate) fn is_aligned_and_not_null<T>(ptr: *const T) -> bool {
     !ptr.is_null() && ptr.is_aligned()
 }
 
+/// Checks whether an allocation of `len` instances of `T` exceeds
+/// the maximum allowed allocation size.
+pub(crate) fn is_valid_allocation_size<T>(len: usize) -> bool {
+    let max_len = const {
+        let size = crate::mem::size_of::<T>();
+        if size == 0 { usize::MAX } else { isize::MAX as usize / size }
+    };
+    len <= max_len
+}
+
 /// Checks whether the regions of memory starting at `src` and `dst` of size
 /// `count * size_of::<T>()` do *not* overlap.
 pub(crate) fn is_nonoverlapping<T>(src: *const T, dst: *const T, count: usize) -> bool {
diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 9cbfbbb9f39..659409557c9 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -192,6 +192,7 @@
 #![feature(extern_types)]
 #![feature(fundamental)]
 #![feature(if_let_guard)]
+#![feature(inline_const)]
 #![feature(intra_doc_pointers)]
 #![feature(intrinsics)]
 #![feature(lang_items)]
diff --git a/library/core/src/slice/raw.rs b/library/core/src/slice/raw.rs
index 3c5abd215a4..dace748fed4 100644
--- a/library/core/src/slice/raw.rs
+++ b/library/core/src/slice/raw.rs
@@ -1,7 +1,9 @@
 //! Free functions to create `&[T]` and `&mut [T]`.
 
 use crate::array;
-use crate::intrinsics::{assert_unsafe_precondition, is_aligned_and_not_null};
+use crate::intrinsics::{
+    assert_unsafe_precondition, is_aligned_and_not_null, is_valid_allocation_size,
+};
 use crate::ops::Range;
 use crate::ptr;
 
@@ -91,8 +93,7 @@ pub const unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T]
     // SAFETY: the caller must uphold the safety contract for `from_raw_parts`.
     unsafe {
         assert_unsafe_precondition!([T](data: *const T, len: usize) =>
-            is_aligned_and_not_null(data)
-                && crate::mem::size_of::<T>().saturating_mul(len) <= isize::MAX as usize
+            is_aligned_and_not_null(data) && is_valid_allocation_size::<T>(len)
         );
         &*ptr::slice_from_raw_parts(data, len)
     }
@@ -135,8 +136,7 @@ pub const unsafe fn from_raw_parts_mut<'a, T>(data: *mut T, len: usize) -> &'a m
     // SAFETY: the caller must uphold the safety contract for `from_raw_parts_mut`.
     unsafe {
         assert_unsafe_precondition!([T](data: *mut T, len: usize) =>
-            is_aligned_and_not_null(data)
-                && crate::mem::size_of::<T>().saturating_mul(len) <= isize::MAX as usize
+            is_aligned_and_not_null(data) && is_valid_allocation_size::<T>(len)
         );
         &mut *ptr::slice_from_raw_parts_mut(data, len)
     }