From 2456495a260827217d3c612d6c577c2f165c61eb Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sun, 30 May 2021 10:25:41 -0700 Subject: Stop generating `alloca`s+`memcmp` for simple array equality --- compiler/rustc_codegen_llvm/src/context.rs | 5 +++++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f662887abf8..d1aecd32e2f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -500,6 +500,7 @@ impl CodegenCx<'b, 'tcx> { let t_i32 = self.type_i32(); let t_i64 = self.type_i64(); let t_i128 = self.type_i128(); + let t_isize = self.type_isize(); let t_f32 = self.type_f32(); let t_f64 = self.type_f64(); @@ -712,6 +713,10 @@ impl CodegenCx<'b, 'tcx> { ifn!("llvm.assume", fn(i1) -> void); ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + // This isn't an "LLVM intrinsic", but LLVM's optimization passes + // recognize it like one and we assume it exists in `core::slice::cmp` + ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32); + // variadic intrinsics ifn!("llvm.va_start", fn(i8p) -> void); ifn!("llvm.va_end", fn(i8p) -> void); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1fb201eda6b..615295e96e1 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -296,6 +296,32 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } + sym::raw_eq => { + let tp_ty = substs.type_at(0); + let (size, align) = self.size_and_align_of(tp_ty); + let a = args[0].immediate(); + let b = args[1].immediate(); + if size.bytes() == 0 { + self.const_bool(true) + } else if size > self.data_layout().pointer_size * 4 { + let i8p_ty = self.type_i8p(); + let a_ptr = self.bitcast(a, i8p_ty); + let b_ptr = self.bitcast(b, i8p_ty); + let n = self.const_usize(size.bytes()); + let llfn = self.get_intrinsic("memcmp"); + let cmp = self.call(llfn, &[a_ptr, b_ptr, n], None); + self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) + } else { + let integer_ty = self.type_ix(size.bits()); + let ptr_ty = self.type_ptr_to(integer_ty); + let a_ptr = self.bitcast(a, ptr_ty); + let a_val = self.load(a_ptr, align); + let b_ptr = self.bitcast(b, ptr_ty); + let b_val = self.load(b_ptr, align); + self.icmp(IntPredicate::IntEQ, a_val, b_val) + } + } + _ if name_str.starts_with("simd_") => { match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { Ok(llval) => llval, -- cgit 1.4.1-3-g733a5 From 07fb5ee78f4f251637c5c4414982a8c6e32e186d Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Wed, 2 Jun 2021 23:35:30 -0700 Subject: Adjust the threshold to look at the ABI, not just the size --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 36 ++++++++++++++++++---------- src/test/codegen/array-equality.rs | 12 ++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 615295e96e1..9a968659e2f 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -297,28 +297,40 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { } sym::raw_eq => { + use abi::Abi::*; let tp_ty = substs.type_at(0); - let (size, align) = self.size_and_align_of(tp_ty); + let layout = self.layout_of(tp_ty).layout; + let use_integer_compare = match layout.abi { + Scalar(_) | ScalarPair(_, _) => true, + Uninhabited | Vector { .. } => false, + Aggregate { .. } => { + // For rusty ABIs, small aggregates are actually passed + // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), + // so we re-use that same threshold here. + layout.size <= self.data_layout().pointer_size * 2 + } + }; + let a = args[0].immediate(); let b = args[1].immediate(); - if size.bytes() == 0 { + if layout.size.bytes() == 0 { self.const_bool(true) - } else if size > self.data_layout().pointer_size * 4 { + } else if use_integer_compare { + let integer_ty = self.type_ix(layout.size.bits()); + let ptr_ty = self.type_ptr_to(integer_ty); + let a_ptr = self.bitcast(a, ptr_ty); + let a_val = self.load(a_ptr, layout.align.abi); + let b_ptr = self.bitcast(b, ptr_ty); + let b_val = self.load(b_ptr, layout.align.abi); + self.icmp(IntPredicate::IntEQ, a_val, b_val) + } else { let i8p_ty = self.type_i8p(); let a_ptr = self.bitcast(a, i8p_ty); let b_ptr = self.bitcast(b, i8p_ty); - let n = self.const_usize(size.bytes()); + let n = self.const_usize(layout.size.bytes()); let llfn = self.get_intrinsic("memcmp"); let cmp = self.call(llfn, &[a_ptr, b_ptr, n], None); self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) - } else { - let integer_ty = self.type_ix(size.bits()); - let ptr_ty = self.type_ptr_to(integer_ty); - let a_ptr = self.bitcast(a, ptr_ty); - let a_val = self.load(a_ptr, align); - let b_ptr = self.bitcast(b, ptr_ty); - let b_val = self.load(b_ptr, align); - self.icmp(IntPredicate::IntEQ, a_val, b_val) } } diff --git a/src/test/codegen/array-equality.rs b/src/test/codegen/array-equality.rs index aa56e32e26c..4b60fa4b0bf 100644 --- a/src/test/codegen/array-equality.rs +++ b/src/test/codegen/array-equality.rs @@ -23,6 +23,18 @@ pub fn array_eq_ref(a: &[u16; 6], b: &[u16; 6]) -> bool { a == b } +// CHECK-LABEL: @array_eq_value_still_passed_by_pointer +#[no_mangle] +pub fn array_eq_value_still_passed_by_pointer(a: [u16; 9], b: [u16; 9]) -> bool { + // CHECK-NEXT: start: + // CHECK-NEXT: bitcast + // CHECK-NEXT: bitcast + // CHECK-NEXT: %[[CMP:.+]] = tail call i32 @{{bcmp|memcmp}}(i8* nonnull dereferenceable(18) %{{.+}}, i8* nonnull dereferenceable(18) %{{.+}}, i64 18) + // CHECK-NEXT: %[[EQ:.+]] = icmp eq i32 %[[CMP]], 0 + // CHECK-NEXT: ret i1 %[[EQ]] + a == b +} + // CHECK-LABEL: @array_eq_long #[no_mangle] pub fn array_eq_long(a: &[u16; 1234], b: &[u16; 1234]) -> bool { -- cgit 1.4.1-3-g733a5