Stop using uadd.with.overflow

3 files changed, 41 insertions, 23 deletions

diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 32cdef075e7..917d07e3c61 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -557,13 +557,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         let (size, signed) = ty.int_size_and_signed(self.tcx);
         let width = size.bits();
 
-        if oop == OverflowOp::Sub && !signed {
-            // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
-            // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
-            // in the backend if profitable.
-            let sub = self.sub(lhs, rhs);
-            let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
-            return (sub, cmp);
+        if !signed {
+            match oop {
+                OverflowOp::Sub => {
+                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
+                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
+                    // in the backend if profitable.
+                    let sub = self.sub(lhs, rhs);
+                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
+                    return (sub, cmp);
+                }
+                OverflowOp::Add => {
+                    // Like with sub above, using icmp is the preferred form. See
+                    // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085>
+                    let add = self.add(lhs, rhs);
+                    let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
+                    return (add, cmp);
+                }
+                OverflowOp::Mul => {}
+            }
         }
 
         let oop_str = match oop {
diff --git a/tests/assembly-llvm/x86_64-bigint-helpers.rs b/tests/assembly-llvm/x86_64-bigint-helpers.rs
index 58785932bc2..c5efda58fd6 100644
--- a/tests/assembly-llvm/x86_64-bigint-helpers.rs
+++ b/tests/assembly-llvm/x86_64-bigint-helpers.rs
@@ -2,9 +2,7 @@
 //@ assembly-output: emit-asm
 //@ compile-flags: --crate-type=lib -Copt-level=3 -C target-cpu=x86-64-v4
 //@ compile-flags: -C llvm-args=-x86-asm-syntax=intel
-//@ revisions: llvm-pre-20 llvm-20
-//@ [llvm-20] min-llvm-version: 20
-//@ [llvm-pre-20] max-llvm-major-version: 19
+//@ min-llvm-version: 20
 
 #![no_std]
 #![feature(bigint_helper_methods)]
@@ -23,16 +21,15 @@ pub unsafe extern "sysv64" fn bigint_chain_carrying_add(
     n: usize,
     mut carry: bool,
 ) -> bool {
-    // llvm-pre-20: mov [[TEMP:r..]], qword ptr [rsi + 8*[[IND:r..]] + 8]
-    // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8]
-    // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP]]
-    // llvm-pre-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 16]
-    // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 16]
-    // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP]]
-    // llvm-20: adc [[TEMP:r..]], qword ptr [rdx + 8*[[IND:r..]]]
-    // llvm-20: mov qword ptr [rdi + 8*[[IND]]], [[TEMP]]
-    // llvm-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 8]
-    // llvm-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8]
+    // Even if we emit A+B, LLVM will sometimes reorder that to B+A, so this
+    // test doesn't actually check which register is mov vs which is adc.
+
+    // CHECK: mov [[TEMP1:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND:.+]] + 8]
+    // CHECK: adc [[TEMP1]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 8]
+    // CHECK: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP1]]
+    // CHECK: mov [[TEMP2:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16]
+    // CHECK: adc [[TEMP2]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16]
+    // CHECK: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP2]]
     for i in 0..n {
         (*dest.add(i), carry) = u64::carrying_add(*src1.add(i), *src2.add(i), carry);
     }
diff --git a/tests/codegen-llvm/bigint-helpers.rs b/tests/codegen-llvm/bigint-helpers.rs
index 355cccb8150..ec70a3eabed 100644
--- a/tests/codegen-llvm/bigint-helpers.rs
+++ b/tests/codegen-llvm/bigint-helpers.rs
@@ -3,11 +3,20 @@
 #![crate_type = "lib"]
 #![feature(bigint_helper_methods)]
 
+// Note that there's also an assembly test for this, which is what checks for
+// the `ADC` (Add with Carry) instruction on x86 now that the IR we emit uses
+// the preferred instruction phrasing instead of the intrinsic.
+
 // CHECK-LABEL: @u32_carrying_add
 #[no_mangle]
 pub fn u32_carrying_add(a: u32, b: u32, c: bool) -> (u32, bool) {
-    // CHECK: @llvm.uadd.with.overflow.i32
-    // CHECK: @llvm.uadd.with.overflow.i32
-    // CHECK: or disjoint i1
+    // CHECK: %[[AB:.+]] = add i32 {{%a, %b|%b, %a}}
+    // CHECK: %[[O1:.+]] = icmp ult i32 %[[AB]], %a
+    // CHECK: %[[CEXT:.+]] = zext i1 %c to i32
+    // CHECK: %[[ABC:.+]] = add i32 %[[AB]], %[[CEXT]]
+    // CHECK: %[[O2:.+]] = icmp ult i32 %[[ABC]], %[[AB]]
+    // CHECK: %[[O:.+]] = or disjoint i1 %[[O1]], %[[O2]]
+    // CHECK: insertvalue {{.+}}, i32 %[[ABC]], 0
+    // CHECK: insertvalue {{.+}}, i1 %[[O]], 1
     u32::carrying_add(a, b, c)
 }