From e6fc62a1ef6cfb545d4f33914a4440c6bbcbf9eb Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Wed, 4 Jul 2018 02:48:30 -0700 Subject: Don't use SIMD in mem::swap for types smaller than the block size LLVM isn't able to remove the alloca for the unaligned block in the SIMD tail in some cases, so doing this helps SRoA work in cases where it currently doesn't. Found in the `replace_with` RFC discussion. --- src/libcore/mem.rs | 2 +- src/libcore/ptr.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src/libcore') diff --git a/src/libcore/mem.rs b/src/libcore/mem.rs index 8fb4e0d6a02..a0fe6e98806 100644 --- a/src/libcore/mem.rs +++ b/src/libcore/mem.rs @@ -638,7 +638,7 @@ pub unsafe fn uninitialized() -> T { #[stable(feature = "rust1", since = "1.0.0")] pub fn swap(x: &mut T, y: &mut T) { unsafe { - ptr::swap_nonoverlapping(x, y, 1); + ptr::swap_nonoverlapping_one(x, y); } } diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index 0af642258c2..f1405b58e1b 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -187,6 +187,19 @@ pub unsafe fn swap_nonoverlapping(x: *mut T, y: *mut T, count: usize) { swap_nonoverlapping_bytes(x, y, len) } +#[inline] +pub(crate) unsafe fn swap_nonoverlapping_one(x: *mut T, y: *mut T) { + // For types smaller than the block optimization below, + // just swap directly to avoid pessimizing codegen. + if mem::size_of::() < 32 { + let z = read(x); + copy_nonoverlapping(y, x, 1); + write(y, z); + } else { + swap_nonoverlapping(x, y, 1); + } +} + #[inline] unsafe fn swap_nonoverlapping_bytes(x: *mut u8, y: *mut u8, len: usize) { // The approach here is to utilize simd to swap x & y efficiently. Testing reveals -- cgit 1.4.1-3-g733a5