From f23025305f88cf4d822452960dd707d5abee555f Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sun, 19 Jan 2025 23:38:09 -0800 Subject: Add `unchecked_disjoint_bitor` with fallback intrinsic implementation --- compiler/rustc_hir_analysis/src/check/intrinsic.rs | 2 +- compiler/rustc_span/src/symbol.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'compiler') diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index cf3d4897304..c3404fa1624 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -472,7 +472,7 @@ pub fn check_intrinsic_type( vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize, ), - sym::unchecked_div | sym::unchecked_rem | sym::exact_div => { + sym::unchecked_div | sym::unchecked_rem | sym::exact_div | sym::disjoint_bitor => { (1, 0, vec![param(0), param(0)], param(0)) } sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 7f7b460cf57..e9e62655bb7 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -776,6 +776,7 @@ symbols! { discriminant_kind, discriminant_type, discriminant_value, + disjoint_bitor, dispatch_from_dyn, div, div_assign, -- cgit 1.4.1-3-g733a5 From 4ee1602eab2cdc88172d4a98f927613ab64b4cf0 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Mon, 20 Jan 2025 00:00:44 -0800 Subject: Override `disjoint_or` in the LLVM backend --- compiler/rustc_codegen_llvm/src/builder.rs | 8 ++++++++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 3 +++ compiler/rustc_codegen_ssa/src/mir/intrinsic.rs | 5 +++++ compiler/rustc_codegen_ssa/src/traits/builder.rs | 5 +++++ tests/codegen/bigint-helpers.rs | 13 +++++++++++++ tests/codegen/intrinsics/disjoint_bitor.rs | 20 ++++++++++++++++++++ 6 files changed, 54 insertions(+) create mode 100644 tests/codegen/bigint-helpers.rs create mode 100644 tests/codegen/intrinsics/disjoint_bitor.rs (limited to 'compiler') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 2d007416263..6e1eec4f0fc 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -421,6 +421,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unchecked_umul(x, y) => LLVMBuildNUWMul, } + fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { + unsafe { + let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED); + llvm::LLVMSetIsDisjoint(or, True); + or + } + } + set_math_builder_methods! { fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath), fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath), diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index cc7c5231aca..8d494b0fe62 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1380,6 +1380,9 @@ unsafe extern "C" { pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + // Extra flags on arithmetic + pub fn LLVMSetIsDisjoint(Instr: &Value, IsDisjoint: Bool); + // Memory pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; pub fn LLVMBuildArrayAlloca<'a>( diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 304ac4544ee..6e7fbe62c8d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -225,6 +225,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args[1].val.unaligned_volatile_store(bx, dst); return Ok(()); } + sym::disjoint_bitor => { + let a = args[0].immediate(); + let b = args[1].immediate(); + bx.or_disjoint(a, b) + } sym::exact_div => { let ty = arg_tys[0]; match int_type_width_signed(ty, bx.tcx()) { diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index bbf87a59942..b7dcf16fa2b 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -167,6 +167,11 @@ pub trait BuilderMethods<'a, 'tcx>: fn unchecked_umul(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn and(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn or(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; + /// Defaults to [`Self::or`], but guarantees `(lhs & rhs) == 0` so some backends + /// can emit something more helpful for optimizations. + fn or_disjoint(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value { + self.or(lhs, rhs) + } fn xor(&mut self, lhs: Self::Value, rhs: Self::Value) -> Self::Value; fn neg(&mut self, v: Self::Value) -> Self::Value; fn fneg(&mut self, v: Self::Value) -> Self::Value; diff --git a/tests/codegen/bigint-helpers.rs b/tests/codegen/bigint-helpers.rs new file mode 100644 index 00000000000..355cccb8150 --- /dev/null +++ b/tests/codegen/bigint-helpers.rs @@ -0,0 +1,13 @@ +//@ compile-flags: -C opt-level=3 + +#![crate_type = "lib"] +#![feature(bigint_helper_methods)] + +// CHECK-LABEL: @u32_carrying_add +#[no_mangle] +pub fn u32_carrying_add(a: u32, b: u32, c: bool) -> (u32, bool) { + // CHECK: @llvm.uadd.with.overflow.i32 + // CHECK: @llvm.uadd.with.overflow.i32 + // CHECK: or disjoint i1 + u32::carrying_add(a, b, c) +} diff --git a/tests/codegen/intrinsics/disjoint_bitor.rs b/tests/codegen/intrinsics/disjoint_bitor.rs new file mode 100644 index 00000000000..be9954507b3 --- /dev/null +++ b/tests/codegen/intrinsics/disjoint_bitor.rs @@ -0,0 +1,20 @@ +//@ compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::disjoint_bitor; + +// CHECK-LABEL: @disjoint_bitor_signed +#[no_mangle] +pub unsafe fn disjoint_bitor_signed(x: i32, y: i32) -> i32 { + // CHECK: or disjoint i32 %x, %y + disjoint_bitor(x, y) +} + +// CHECK-LABEL: @disjoint_bitor_unsigned +#[no_mangle] +pub unsafe fn disjoint_bitor_unsigned(x: u64, y: u64) -> u64 { + // CHECK: or disjoint i64 %x, %y + disjoint_bitor(x, y) +} -- cgit 1.4.1-3-g733a5 From f46e6be1908c7ed729655c8f601548f732ef49f4 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sun, 2 Feb 2025 19:23:44 -0800 Subject: Handle the case where the `or disjoint` folds immediately to a constant --- compiler/rustc_codegen_llvm/src/builder.rs | 8 +++++++- tests/codegen/intrinsics/disjoint_bitor.rs | 12 +++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'compiler') diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 6e1eec4f0fc..264d43c6d46 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -424,7 +424,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value { unsafe { let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED); - llvm::LLVMSetIsDisjoint(or, True); + + // If a and b are both values, then `or` is a value, rather than + // an instruction, so we need to check before setting the flag. + // (See also `LLVMBuildNUWNeg` which also needs a check.) + if llvm::LLVMIsAInstruction(or).is_some() { + llvm::LLVMSetIsDisjoint(or, True); + } or } } diff --git a/tests/codegen/intrinsics/disjoint_bitor.rs b/tests/codegen/intrinsics/disjoint_bitor.rs index be9954507b3..fc45439ee0b 100644 --- a/tests/codegen/intrinsics/disjoint_bitor.rs +++ b/tests/codegen/intrinsics/disjoint_bitor.rs @@ -1,4 +1,4 @@ -//@ compile-flags: -C no-prepopulate-passes +//@ compile-flags: -C no-prepopulate-passes -Z mir-opt-level=0 #![crate_type = "lib"] #![feature(core_intrinsics)] @@ -18,3 +18,13 @@ pub unsafe fn disjoint_bitor_unsigned(x: u64, y: u64) -> u64 { // CHECK: or disjoint i64 %x, %y disjoint_bitor(x, y) } + +// CHECK-LABEL: @disjoint_bitor_literal +#[no_mangle] +pub unsafe fn disjoint_bitor_literal() -> u8 { + // This is a separate check because even without any passes, + // LLVM will fold so it's not an instruction, which can assert in LLVM. + + // CHECK: store i8 3 + disjoint_bitor(1, 2) +} -- cgit 1.4.1-3-g733a5