Help LLVM better optimize slice::Iter(Mut)::len

3 files changed, 37 insertions, 4 deletions

diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs
index 8d9a51742fd..b30eff8baa9 100644
--- a/src/libcore/intrinsics.rs
+++ b/src/libcore/intrinsics.rs
@@ -1607,3 +1607,9 @@ pub fn maxnumf64(x: f64, y: f64) -> f64 {
     // Identical to the `f32` case.
     (if x < y || x != x { y } else { x }) * 1.0
 }
+
+/// For bootstrapping, implement unchecked_sub as just wrapping_sub.
+#[cfg(bootstrap)]
+pub unsafe fn unchecked_sub<T>(x: T, y: T) -> T {
+    sub_with_overflow(x, y).0
+}
diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs
index d93e5a9ca2b..b2376cdf9fa 100644
--- a/src/libcore/slice/mod.rs
+++ b/src/libcore/slice/mod.rs
@@ -25,7 +25,7 @@
 use crate::cmp::Ordering::{self, Less, Equal, Greater};
 use crate::cmp;
 use crate::fmt;
-use crate::intrinsics::assume;
+use crate::intrinsics::{assume, exact_div, unchecked_sub};
 use crate::isize;
 use crate::iter::*;
 use crate::ops::{FnMut, Try, self};
@@ -2998,14 +2998,27 @@ macro_rules! is_empty {
 // unexpected way. (Tested by `codegen/slice-position-bounds-check`.)
 macro_rules! len {
     ($self: ident) => {{
+        #![allow(unused_unsafe)] // we're sometimes used within an unsafe block
+
         let start = $self.ptr;
-        let diff = ($self.end as usize).wrapping_sub(start as usize);
         let size = size_from_ptr(start);
         if size == 0 {
+            // This _cannot_ use `unchecked_sub` because we depend on wrapping
+            // to represent the length of long ZST slice iterators.
+            let diff = ($self.end as usize).wrapping_sub(start as usize);
             diff
         } else {
-            // Using division instead of `offset_from` helps LLVM remove bounds checks
-            diff / size
+            // We know that `start <= end`, so can do better than `offset_from`,
+            // which needs to deal in signed.  By setting appropriate flags here
+            // we can tell LLVM this, which helps it remove bounds checks.
+            // SAFETY: By the type invariant, `start <= end`
+            let diff = unsafe { unchecked_sub($self.end as usize, start as usize) };
+            // By also telling LLVM that the pointers are apart by an exact
+            // multiple of the type size, it can optimize `len() == 0` down to
+            // `start == end` instead of `(end - start) < size`.
+            // SAFETY: By the type invariant, the pointers are aligned so the
+            //         distance between them must be a multiple of pointee size
+            unsafe { exact_div(diff, size) }
         }
     }}
 }
diff --git a/src/test/codegen/slice-iter-len-eq-zero.rs b/src/test/codegen/slice-iter-len-eq-zero.rs
new file mode 100644
index 00000000000..a5516833900
--- /dev/null
+++ b/src/test/codegen/slice-iter-len-eq-zero.rs
@@ -0,0 +1,14 @@
+// no-system-llvm
+// compile-flags: -O
+#![crate_type = "lib"]
+
+type Demo = [u8; 3];
+
+// CHECK-LABEL: @slice_iter_len_eq_zero
+#[no_mangle]
+pub fn slice_iter_len_eq_zero(y: std::slice::Iter<'_, Demo>) -> bool {
+    // CHECK-NOT: sub
+    // CHECK: %2 = icmp eq i8* %1, %0
+    // CHECK: ret i1 %2
+    y.len() == 0
+}