Support memcpy/memmove with differing src/dst alignment

If LLVM 7 is used, generate memcpy/memmove with differing
src/dst alignment. I've added new FFI functions to construct
these through the builder API, which is more convenient than
dealing with differing intrinsic signatures depending on the
LLVM version.

11 files changed, 98 insertions, 51 deletions

diff --git a/src/librustc_codegen_llvm/abi.rs b/src/librustc_codegen_llvm/abi.rs
index e50534a4e1d..bd7a1c2c329 100644
--- a/src/librustc_codegen_llvm/abi.rs
+++ b/src/librustc_codegen_llvm/abi.rs
@@ -225,9 +225,10 @@ impl ArgTypeExt<'ll, 'tcx> for ArgType<'tcx, Ty<'tcx>> {
                 // ...and then memcpy it to the intended destination.
                 base::call_memcpy(bx,
                                   bx.pointercast(dst.llval, Type::i8p(cx)),
+                                  self.layout.align,
                                   bx.pointercast(llscratch, Type::i8p(cx)),
+                                  scratch_align,
                                   C_usize(cx, self.layout.size.bytes()),
-                                  self.layout.align.min(scratch_align),
                                   MemFlags::empty());
 
                 bx.lifetime_end(llscratch, scratch_size);
diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs
index 9736994fc07..dd1348cdf2b 100644
--- a/src/librustc_codegen_llvm/base.rs
+++ b/src/librustc_codegen_llvm/base.rs
@@ -53,7 +53,7 @@ use mir::place::PlaceRef;
 use attributes;
 use builder::{Builder, MemFlags};
 use callee;
-use common::{C_bool, C_bytes_in_context, C_i32, C_usize};
+use common::{C_bool, C_bytes_in_context, C_usize};
 use rustc_mir::monomorphize::item::DefPathBasedNames;
 use common::{C_struct_in_context, C_array, val_ty};
 use consts;
@@ -77,7 +77,6 @@ use rustc_data_structures::sync::Lrc;
 use std::any::Any;
 use std::cmp;
 use std::ffi::CString;
-use std::i32;
 use std::ops::{Deref, DerefMut};
 use std::sync::mpsc;
 use std::time::{Instant, Duration};
@@ -319,8 +318,8 @@ pub fn coerce_unsized_into(
                 }
 
                 if src_f.layout.ty == dst_f.layout.ty {
-                    memcpy_ty(bx, dst_f.llval, src_f.llval, src_f.layout,
-                              src_f.align.min(dst_f.align), MemFlags::empty());
+                    memcpy_ty(bx, dst_f.llval, dst_f.align, src_f.llval, src_f.align,
+                              src_f.layout, MemFlags::empty());
                 } else {
                     coerce_unsized_into(bx, src_f, dst_f);
                 }
@@ -420,36 +419,34 @@ pub fn to_immediate_scalar(
 pub fn call_memcpy(
     bx: &Builder<'_, 'll, '_>,
     dst: &'ll Value,
+    dst_align: Align,
     src: &'ll Value,
+    src_align: Align,
     n_bytes: &'ll Value,
-    align: Align,
     flags: MemFlags,
 ) {
     if flags.contains(MemFlags::NONTEMPORAL) {
         // HACK(nox): This is inefficient but there is no nontemporal memcpy.
-        let val = bx.load(src, align);
+        let val = bx.load(src, src_align);
         let ptr = bx.pointercast(dst, val_ty(val).ptr_to());
-        bx.store_with_flags(val, ptr, align, flags);
+        bx.store_with_flags(val, ptr, dst_align, flags);
         return;
     }
     let cx = bx.cx;
-    let ptr_width = &cx.sess().target.target.target_pointer_width;
-    let key = format!("llvm.memcpy.p0i8.p0i8.i{}", ptr_width);
-    let memcpy = cx.get_intrinsic(&key);
     let src_ptr = bx.pointercast(src, Type::i8p(cx));
     let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
     let size = bx.intcast(n_bytes, cx.isize_ty, false);
-    let align = C_i32(cx, align.abi() as i32);
-    let volatile = C_bool(cx, flags.contains(MemFlags::VOLATILE));
-    bx.call(memcpy, &[dst_ptr, src_ptr, size, align, volatile], None);
+    let volatile = flags.contains(MemFlags::VOLATILE);
+    bx.memcpy(dst_ptr, dst_align.abi(), src_ptr, src_align.abi(), size, volatile);
 }
 
 pub fn memcpy_ty(
     bx: &Builder<'_, 'll, 'tcx>,
     dst: &'ll Value,
+    dst_align: Align,
     src: &'ll Value,
+    src_align: Align,
     layout: TyLayout<'tcx>,
-    align: Align,
     flags: MemFlags,
 ) {
     let size = layout.size.bytes();
@@ -457,7 +454,7 @@ pub fn memcpy_ty(
         return;
     }
 
-    call_memcpy(bx, dst, src, C_usize(bx.cx, size), align, flags);
+    call_memcpy(bx, dst, dst_align, src, src_align, C_usize(bx.cx, size), flags);
 }
 
 pub fn call_memset(
diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs
index 2fe6a0377f8..eaaa450fbf4 100644
--- a/src/librustc_codegen_llvm/builder.rs
+++ b/src/librustc_codegen_llvm/builder.rs
@@ -784,6 +784,24 @@ impl Builder<'a, 'll, 'tcx> {
         }
     }
 
+    pub fn memcpy(&self, dst: &'ll Value, dst_align: u64,
+                  src: &'ll Value, src_align: u64,
+                  size: &'ll Value, is_volatile: bool) -> &'ll Value {
+        unsafe {
+            llvm::LLVMRustBuildMemCpy(self.llbuilder, dst, dst_align as c_uint,
+                                      src, src_align as c_uint, size, is_volatile)
+        }
+    }
+
+    pub fn memmove(&self, dst: &'ll Value, dst_align: u64,
+                  src: &'ll Value, src_align: u64,
+                  size: &'ll Value, is_volatile: bool) -> &'ll Value {
+        unsafe {
+            llvm::LLVMRustBuildMemMove(self.llbuilder, dst, dst_align as c_uint,
+                                      src, src_align as c_uint, size, is_volatile)
+        }
+    }
+
     pub fn minnum(&self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         self.count_insn("minnum");
         unsafe {
diff --git a/src/librustc_codegen_llvm/context.rs b/src/librustc_codegen_llvm/context.rs
index 241f7989e16..59dc0f61ecc 100644
--- a/src/librustc_codegen_llvm/context.rs
+++ b/src/librustc_codegen_llvm/context.rs
@@ -530,12 +530,6 @@ fn declare_intrinsic(cx: &CodegenCx<'ll, '_>, key: &str) -> Option<&'ll Value> {
     let t_v4f64 = Type::vector(t_f64, 4);
     let t_v8f64 = Type::vector(t_f64, 8);
 
-    ifn!("llvm.memcpy.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
-    ifn!("llvm.memcpy.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
-    ifn!("llvm.memcpy.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
-    ifn!("llvm.memmove.p0i8.p0i8.i16", fn(i8p, i8p, t_i16, t_i32, i1) -> void);
-    ifn!("llvm.memmove.p0i8.p0i8.i32", fn(i8p, i8p, t_i32, t_i32, i1) -> void);
-    ifn!("llvm.memmove.p0i8.p0i8.i64", fn(i8p, i8p, t_i64, t_i32, i1) -> void);
     ifn!("llvm.memset.p0i8.i16", fn(i8p, t_i8, t_i16, t_i32, i1) -> void);
     ifn!("llvm.memset.p0i8.i32", fn(i8p, t_i8, t_i32, t_i32, i1) -> void);
     ifn!("llvm.memset.p0i8.i64", fn(i8p, t_i8, t_i64, t_i32, i1) -> void);
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs
index 03244c18ac3..f7cef6be5b5 100644
--- a/src/librustc_codegen_llvm/intrinsic.rs
+++ b/src/librustc_codegen_llvm/intrinsic.rs
@@ -23,7 +23,7 @@ use glue;
 use type_::Type;
 use type_of::LayoutLlvmExt;
 use rustc::ty::{self, Ty};
-use rustc::ty::layout::{HasDataLayout, LayoutOf};
+use rustc::ty::layout::LayoutOf;
 use rustc::hir;
 use syntax::ast;
 use syntax::symbol::Symbol;
@@ -690,28 +690,14 @@ fn copy_intrinsic(
     let cx = bx.cx;
     let (size, align) = cx.size_and_align_of(ty);
     let size = C_usize(cx, size.bytes());
-    let align = C_i32(cx, align.abi() as i32);
-
-    let operation = if allow_overlap {
-        "memmove"
-    } else {
-        "memcpy"
-    };
-
-    let name = format!("llvm.{}.p0i8.p0i8.i{}", operation,
-                       cx.data_layout().pointer_size.bits());
-
+    let align = align.abi();
     let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
     let src_ptr = bx.pointercast(src, Type::i8p(cx));
-    let llfn = cx.get_intrinsic(&name);
-
-    bx.call(llfn,
-        &[dst_ptr,
-        src_ptr,
-        bx.mul(size, count),
-        align,
-        C_bool(cx, volatile)],
-        None)
+    if allow_overlap {
+        bx.memmove(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
+    } else {
+        bx.memcpy(dst_ptr, align, src_ptr, align, bx.mul(size, count), volatile)
+    }
 }
 
 fn memset_intrinsic(
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index f046ea03027..64066d4d786 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -998,6 +998,22 @@ extern "C" {
                              Bundle: Option<&OperandBundleDef<'a>>,
                              Name: *const c_char)
                              -> &'a Value;
+    pub fn LLVMRustBuildMemCpy(B: &Builder<'a>,
+                               Dst: &'a Value,
+                               DstAlign: c_uint,
+                               Src: &'a Value,
+                               SrcAlign: c_uint,
+                               Size: &'a Value,
+                               IsVolatile: bool)
+                               -> &'a Value;
+    pub fn LLVMRustBuildMemMove(B: &Builder<'a>,
+                                Dst: &'a Value,
+                                DstAlign: c_uint,
+                                Src: &'a Value,
+                                SrcAlign: c_uint,
+                                Size: &'a Value,
+                                IsVolatile: bool)
+                                -> &'a Value;
     pub fn LLVMBuildSelect(B: &Builder<'a>,
                            If: &'a Value,
                            Then: &'a Value,
diff --git a/src/librustc_codegen_llvm/mir/block.rs b/src/librustc_codegen_llvm/mir/block.rs
index a7f4c48c89b..3f9921a5cf9 100644
--- a/src/librustc_codegen_llvm/mir/block.rs
+++ b/src/librustc_codegen_llvm/mir/block.rs
@@ -784,7 +784,8 @@ impl FunctionCx<'a, 'll, 'tcx> {
                     // have scary latent bugs around.
 
                     let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
-                    base::memcpy_ty(bx, scratch.llval, llval, op.layout, align, MemFlags::empty());
+                    base::memcpy_ty(bx, scratch.llval, scratch.align, llval, align,
+                                    op.layout, MemFlags::empty());
                     (scratch.llval, scratch.align, true)
                 } else {
                     (llval, align, true)
diff --git a/src/librustc_codegen_llvm/mir/operand.rs b/src/librustc_codegen_llvm/mir/operand.rs
index d1b6aa7fc42..c76cbfcd971 100644
--- a/src/librustc_codegen_llvm/mir/operand.rs
+++ b/src/librustc_codegen_llvm/mir/operand.rs
@@ -282,8 +282,8 @@ impl OperandValue<'ll> {
         }
         match self {
             OperandValue::Ref(r, None, source_align) => {
-                base::memcpy_ty(bx, dest.llval, r, dest.layout,
-                                source_align.min(dest.align), flags)
+                base::memcpy_ty(bx, dest.llval, dest.align, r, source_align,
+                                dest.layout, flags)
             }
             OperandValue::Ref(_, Some(_), _) => {
                 bug!("cannot directly store unsized values");
@@ -324,7 +324,7 @@ impl OperandValue<'ll> {
         // Allocate an appropriate region on the stack, and copy the value into it
         let (llsize, _) = glue::size_and_align_of_dst(&bx, unsized_ty, Some(llextra));
         let lldst = bx.array_alloca(Type::i8(bx.cx), llsize, "unsized_tmp", max_align);
-        base::call_memcpy(&bx, lldst, llptr, llsize, min_align, flags);
+        base::call_memcpy(&bx, lldst, max_align, llptr, min_align, llsize, flags);
 
         // Store the allocated region and the extra to the indirect place.
         let indirect_operand = OperandValue::Pair(lldst, llextra);
diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp
index affec73e3ac..3a1e082b755 100644
--- a/src/rustllvm/RustWrapper.cpp
+++ b/src/rustllvm/RustWrapper.cpp
@@ -1239,6 +1239,40 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
       unwrap(Fn), makeArrayRef(unwrap(Args), NumArgs), Bundles, Name));
 }
 
+extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B,
+                                            LLVMValueRef Dst, unsigned DstAlign,
+                                            LLVMValueRef Src, unsigned SrcAlign,
+                                            LLVMValueRef Size, bool IsVolatile) {
+#if LLVM_VERSION_GE(7, 0)
+  return wrap(unwrap(B)->CreateMemCpy(
+      unwrap(Dst), DstAlign,
+      unwrap(Src), SrcAlign,
+      unwrap(Size), IsVolatile));
+#else
+  unsigned Align = std::min(DstAlign, SrcAlign);
+  return wrap(unwrap(B)->CreateMemCpy(
+      unwrap(Dst), unwrap(Src),
+      unwrap(Size), Align, IsVolatile));
+#endif
+}
+
+extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
+                                             LLVMValueRef Dst, unsigned DstAlign,
+                                             LLVMValueRef Src, unsigned SrcAlign,
+                                             LLVMValueRef Size, bool IsVolatile) {
+#if LLVM_VERSION_GE(7, 0)
+  return wrap(unwrap(B)->CreateMemMove(
+      unwrap(Dst), DstAlign,
+      unwrap(Src), SrcAlign,
+      unwrap(Size), IsVolatile));
+#else
+  unsigned Align = std::min(DstAlign, SrcAlign);
+  return wrap(unwrap(B)->CreateMemMove(
+      unwrap(Dst), unwrap(Src),
+      unwrap(Size), Align, IsVolatile));
+#endif
+}
+
 extern "C" LLVMValueRef
 LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args,
                     unsigned NumArgs, LLVMBasicBlockRef Then,
diff --git a/src/test/codegen/packed.rs b/src/test/codegen/packed.rs
index 10dd12909b6..b50f5b6f16f 100644
--- a/src/test/codegen/packed.rs
+++ b/src/test/codegen/packed.rs
@@ -65,7 +65,7 @@ pub struct BigPacked2 {
 pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
 // CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
 // CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
-// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 1 %{{.*}}, i{{[0-9]+}} 32, i1 false)
+// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
     // check that calls whose destination is a field of a packed struct
     // go through an alloca rather than calling the function with an
     // unaligned destination.
@@ -77,7 +77,7 @@ pub fn call_pkd1(f: fn() -> Array) -> BigPacked1 {
 pub fn call_pkd2(f: fn() -> Array) -> BigPacked2 {
 // CHECK: [[ALLOCA:%[_a-z0-9]+]] = alloca %Array
 // CHECK: call void %{{.*}}(%Array* noalias nocapture sret dereferenceable(32) [[ALLOCA]])
-// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 2 %{{.*}}, i{{[0-9]+}} 32, i1 false)
+// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 2 %{{.*}}, i8* align 4 %{{.*}}, i{{[0-9]+}} 32, i1 false)
     // check that calls whose destination is a field of a packed struct
     // go through an alloca rather than calling the function with an
     // unaligned destination.
diff --git a/src/test/codegen/stores.rs b/src/test/codegen/stores.rs
index 0aaf00bfdbe..871bee13b19 100644
--- a/src/test/codegen/stores.rs
+++ b/src/test/codegen/stores.rs
@@ -31,7 +31,7 @@ pub fn small_array_alignment(x: &mut [i8; 4], y: [i8; 4]) {
 // CHECK: store i32 %0, i32* [[TMP]]
 // CHECK: [[Y8:%[0-9]+]] = bitcast [4 x i8]* %y to i8*
 // CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8*
-// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false)
+// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false)
     *x = y;
 }
 
@@ -45,6 +45,6 @@ pub fn small_struct_alignment(x: &mut Bytes, y: Bytes) {
 // CHECK: store i32 %0, i32* [[TMP]]
 // CHECK: [[Y8:%[0-9]+]] = bitcast %Bytes* %y to i8*
 // CHECK: [[TMP8:%[0-9]+]] = bitcast i32* [[TMP]] to i8*
-// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 1 [[TMP8]], i{{[0-9]+}} 4, i1 false)
+// CHECK: call void @llvm.memcpy.{{.*}}(i8* align 1 [[Y8]], i8* align 4 [[TMP8]], i{{[0-9]+}} 4, i1 false)
     *x = y;
 }