diff options
| author | bors <bors@rust-lang.org> | 2017-05-10 08:54:50 +0000 |
|---|---|---|
| committer | bors <bors@rust-lang.org> | 2017-05-10 08:54:50 +0000 |
| commit | 2b97174ada7fb1854269558ed2cf3b089e58beee (patch) | |
| tree | 880576d0244be04f587f845d2cb96936d213f3ff /src/libcore | |
| parent | 58b33ad70cdd11f9ce7b5874c6effab9627e51aa (diff) | |
| parent | da91361d2a8ea86a42cbe2a23a7ff816cc5500af (diff) | |
Auto merge of #41764 - scottmcm:faster-reverse, r=brson
Make [u8]::reverse() 5x faster Since LLVM doesn't vectorize the loop for us, do unaligned reads of a larger type and use LLVM's bswap intrinsic to do the reversing of the actual bytes. cfg!-restricted to x86 and x86_64, as I assume it wouldn't help on things like ARMv5. Also makes [u16]::reverse() a more modest 1.5x faster by loading/storing u32 and swapping the u16s with ROT16. Thank you ptr::*_unaligned for making this easy :) Benchmark results (from my i5-2500K): ```text # Before test slice::reverse_u8 ... bench: 273,836 ns/iter (+/- 15,592) = 3829 MB/s test slice::reverse_u16 ... bench: 139,793 ns/iter (+/- 17,748) = 7500 MB/s test slice::reverse_u32 ... bench: 74,997 ns/iter (+/- 5,130) = 13981 MB/s test slice::reverse_u64 ... bench: 47,452 ns/iter (+/- 2,213) = 22097 MB/s # After test slice::reverse_u8 ... bench: 52,170 ns/iter (+/- 3,962) = 20099 MB/s test slice::reverse_u16 ... bench: 93,330 ns/iter (+/- 4,412) = 11235 MB/s test slice::reverse_u32 ... bench: 74,731 ns/iter (+/- 1,425) = 14031 MB/s test slice::reverse_u64 ... bench: 47,556 ns/iter (+/- 3,025) = 22049 MB/s ``` If you're curious about the assembly, instead of doing this ``` movzx eax, byte ptr [rdi] movzx ecx, byte ptr [rsi] mov byte ptr [rdi], cl mov byte ptr [rsi], al ``` it does this ``` mov rax, qword ptr [rdx] mov rbx, qword ptr [r11 + rcx - 8] bswap rbx mov qword ptr [rdx], rbx bswap rax mov qword ptr [r11 + rcx - 8], rax ```
Diffstat (limited to 'src/libcore')
| -rw-r--r-- | src/libcore/slice/mod.rs | 49 |
1 files changed, 49 insertions, 0 deletions
diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index 9e3bd911546..e15eb8f2444 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -539,6 +539,55 @@ impl<T> SliceExt for [T] { fn reverse(&mut self) { let mut i: usize = 0; let ln = self.len(); + + // For very small types, all the individual reads in the normal + // path perform poorly. We can do better, given efficient unaligned + // load/store, by loading a larger chunk and reversing a register. + + // Ideally LLVM would do this for us, as it knows better than we do + // whether unaligned reads are efficient (since that changes between + // different ARM versions, for example) and what the best chunk size + // would be. Unfortunately, as of LLVM 4.0 (2017-05) it only unrolls + // the loop, so we need to do this ourselves. (Hypothesis: reverse + // is troublesome because the sides can be aligned differently -- + // will be, when the length is odd -- so there's no way of emitting + // pre- and postludes to use fully-aligned SIMD in the middle.) + + let fast_unaligned = + cfg!(any(target_arch = "x86", target_arch = "x86_64")); + + if fast_unaligned && mem::size_of::<T>() == 1 { + // Use the llvm.bswap intrinsic to reverse u8s in a usize + let chunk = mem::size_of::<usize>(); + while i + chunk - 1 < ln / 2 { + unsafe { + let pa: *mut T = self.get_unchecked_mut(i); + let pb: *mut T = self.get_unchecked_mut(ln - i - chunk); + let va = ptr::read_unaligned(pa as *mut usize); + let vb = ptr::read_unaligned(pb as *mut usize); + ptr::write_unaligned(pa as *mut usize, vb.swap_bytes()); + ptr::write_unaligned(pb as *mut usize, va.swap_bytes()); + } + i += chunk; + } + } + + if fast_unaligned && mem::size_of::<T>() == 2 { + // Use rotate-by-16 to reverse u16s in a u32 + let chunk = mem::size_of::<u32>() / 2; + while i + chunk - 1 < ln / 2 { + unsafe { + let pa: *mut T = self.get_unchecked_mut(i); + let pb: *mut T = self.get_unchecked_mut(ln - i - chunk); + let va = ptr::read_unaligned(pa as *mut u32); + let vb = ptr::read_unaligned(pb as *mut u32); + ptr::write_unaligned(pa as *mut u32, vb.rotate_left(16)); + ptr::write_unaligned(pb as *mut u32, va.rotate_left(16)); + } + i += chunk; + } + } + while i < ln / 2 { // Unsafe swap to avoid the bounds check in safe swap. unsafe { |
