about summary refs log tree commit diff
diff options
context:
space:
mode:
authorbors <bors@rust-lang.org>2022-05-02 21:45:20 +0000
committerbors <bors@rust-lang.org>2022-05-02 21:45:20 +0000
commit3d0ac7ea23888438752957eeeb5aa2b73b4fda72 (patch)
treefc7a1602a1d73a0e5a9eed9b28fbc90c42ffdbc1
parentbed05e996e37e44b1a3980b84754af621fd3c4ce (diff)
parenta68a5d219d0138c7b39403c6b9dc08e2cf5d05be (diff)
downloadrust-3d0ac7ea23888438752957eeeb5aa2b73b4fda72.tar.gz
rust-3d0ac7ea23888438752957eeeb5aa2b73b4fda72.zip
Auto merge of #92566 - the8472:inline-tra, r=m-ou-se
Inline `__iterator_get_unchecked` for some iterator adapters.

This aligns the inline attributes of existing `__iterator_get_unchecked` with those of `next()` on adapters that have both.

It improves the performance of iterators using unchecked access when building in incremental mode (due to the larger CGU count?). It might negatively affect incremental compile times for better runtime results, but considering that the equivalent `next()` implementations also are `#[inline]` and usually are more complex this should be ok.

```
./x.py bench library/core -i --stage 0 --test-args bench_trusted_random_access

OLD: 119,172 ns/iter
NEW:  17,714 ns/iter
```
-rw-r--r--library/core/benches/iter.rs24
-rw-r--r--library/core/benches/lib.rs1
-rw-r--r--library/core/src/iter/adapters/enumerate.rs1
-rw-r--r--library/core/src/iter/adapters/map.rs1
-rw-r--r--library/core/src/iter/adapters/zip.rs2
-rw-r--r--library/core/src/slice/iter/macros.rs1
6 files changed, 30 insertions, 0 deletions
diff --git a/library/core/benches/iter.rs b/library/core/benches/iter.rs
index 24257ba9878..0abe20e4ca3 100644
--- a/library/core/benches/iter.rs
+++ b/library/core/benches/iter.rs
@@ -367,3 +367,27 @@ fn bench_partial_cmp(b: &mut Bencher) {
 fn bench_lt(b: &mut Bencher) {
     b.iter(|| (0..100000).map(black_box).lt((0..100000).map(black_box)))
 }
+
+#[bench]
+fn bench_trusted_random_access_adapters(b: &mut Bencher) {
+    let vec1: Vec<_> = (0usize..100000).collect();
+    let vec2 = black_box(vec1.clone());
+    b.iter(|| {
+        let mut iter = vec1
+            .iter()
+            .copied()
+            .enumerate()
+            .map(|(idx, e)| idx.wrapping_add(e))
+            .zip(vec2.iter().copied())
+            .map(|(a, b)| a.wrapping_add(b))
+            .fuse();
+        let mut acc: usize = 0;
+        let size = iter.size();
+        for i in 0..size {
+            // SAFETY: TRA requirements are satisfied by 0..size iteration and then dropping the
+            // iterator.
+            acc = acc.wrapping_add(unsafe { iter.__iterator_get_unchecked(i) });
+        }
+        acc
+    })
+}
diff --git a/library/core/benches/lib.rs b/library/core/benches/lib.rs
index d5e1ec083f9..f1f1ae6e463 100644
--- a/library/core/benches/lib.rs
+++ b/library/core/benches/lib.rs
@@ -3,6 +3,7 @@
 #![feature(flt2dec)]
 #![feature(int_log)]
 #![feature(test)]
+#![feature(trusted_random_access)]
 
 extern crate test;
 
diff --git a/library/core/src/iter/adapters/enumerate.rs b/library/core/src/iter/adapters/enumerate.rs
index 84e4618844a..10b4db84b39 100644
--- a/library/core/src/iter/adapters/enumerate.rs
+++ b/library/core/src/iter/adapters/enumerate.rs
@@ -129,6 +129,7 @@ where
 
     #[rustc_inherit_overflow_checks]
     #[doc(hidden)]
+    #[inline]
     unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> <Self as Iterator>::Item
     where
         Self: TrustedRandomAccessNoCoerce,
diff --git a/library/core/src/iter/adapters/map.rs b/library/core/src/iter/adapters/map.rs
index 4b03449972c..6cbb35dc7c6 100644
--- a/library/core/src/iter/adapters/map.rs
+++ b/library/core/src/iter/adapters/map.rs
@@ -125,6 +125,7 @@ where
     }
 
     #[doc(hidden)]
+    #[inline]
     unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> B
     where
         Self: TrustedRandomAccessNoCoerce,
diff --git a/library/core/src/iter/adapters/zip.rs b/library/core/src/iter/adapters/zip.rs
index f50e71da20f..de44bd66501 100644
--- a/library/core/src/iter/adapters/zip.rs
+++ b/library/core/src/iter/adapters/zip.rs
@@ -554,6 +554,7 @@ pub unsafe trait TrustedRandomAccessNoCoerce: Sized {
 ///
 /// Same requirements calling `get_unchecked` directly.
 #[doc(hidden)]
+#[inline]
 pub(in crate::iter::adapters) unsafe fn try_get_unchecked<I>(it: &mut I, idx: usize) -> I::Item
 where
     I: Iterator,
@@ -576,6 +577,7 @@ unsafe impl<I: Iterator> SpecTrustedRandomAccess for I {
 }
 
 unsafe impl<I: Iterator + TrustedRandomAccessNoCoerce> SpecTrustedRandomAccess for I {
+    #[inline]
     unsafe fn try_get_unchecked(&mut self, index: usize) -> Self::Item {
         // SAFETY: the caller must uphold the contract for
         // `Iterator::__iterator_get_unchecked`.
diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs
index b74ab28fc09..78bf3381b4d 100644
--- a/library/core/src/slice/iter/macros.rs
+++ b/library/core/src/slice/iter/macros.rs
@@ -326,6 +326,7 @@ macro_rules! iterator {
             }
 
             #[doc(hidden)]
+            #[inline]
             unsafe fn __iterator_get_unchecked(&mut self, idx: usize) -> Self::Item {
                 // SAFETY: the caller must guarantee that `i` is in bounds of
                 // the underlying slice, so `i` cannot overflow an `isize`, and