diff options
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 47 | ||||
| -rw-r--r-- | compiler/rustc_codegen_ssa/src/traits/builder.rs | 21 | ||||
| -rw-r--r-- | tests/codegen/intrinsics/transmute-niched.rs | 122 | ||||
| -rw-r--r-- | tests/codegen/transmute-optimized.rs | 8 |
4 files changed, 114 insertions, 84 deletions
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 31793641d75..40d5abab8e8 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -365,10 +365,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { use abi::Primitive::*; imm = bx.from_immediate(imm); - // When scalars are passed by value, there's no metadata recording their - // valid ranges. For example, `char`s are passed as just `i32`, with no - // way for LLVM to know that they're 0x10FFFF at most. Thus we assume - // the range of the input value too, not just the output range. + // If we have a scalar, we must already know its range. Either + // + // 1) It's a parameter with `range` parameter metadata, + // 2) It's something we `load`ed with `!range` metadata, or + // 3) After a transmute we `assume`d the range (see below). + // + // That said, last time we tried removing this, it didn't actually help + // the rustc-perf results, so might as well keep doing it + // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182> self.assume_scalar_range(bx, imm, from_scalar, from_backend_ty); imm = match (from_scalar.primitive(), to_scalar.primitive()) { @@ -389,7 +394,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.bitcast(int_imm, to_backend_ty) } }; + + // This `assume` remains important for cases like (a conceptual) + // transmute::<u32, NonZeroU32>(x) == 0 + // since it's never passed to something with parameter metadata (especially + // after MIR inlining) so the only way to tell the backend about the + // constraint that the `transmute` introduced is to `assume` it. self.assume_scalar_range(bx, imm, to_scalar, to_backend_ty); + imm = bx.to_immediate_scalar(imm, to_scalar); imm } @@ -411,31 +423,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - let abi::WrappingRange { start, end } = scalar.valid_range(self.cx); - - if start <= end { - if start > 0 { - let low = bx.const_uint_big(backend_ty, start); - let cmp = bx.icmp(IntPredicate::IntUGE, imm, low); - bx.assume(cmp); - } - - let type_max = scalar.size(self.cx).unsigned_int_max(); - if end < type_max { - let high = bx.const_uint_big(backend_ty, end); - let cmp = bx.icmp(IntPredicate::IntULE, imm, high); - bx.assume(cmp); - } - } else { - let low = bx.const_uint_big(backend_ty, start); - let cmp_low = bx.icmp(IntPredicate::IntUGE, imm, low); - - let high = bx.const_uint_big(backend_ty, end); - let cmp_high = bx.icmp(IntPredicate::IntULE, imm, high); - - let or = bx.or(cmp_low, cmp_high); - bx.assume(or); - } + let range = scalar.valid_range(self.cx); + bx.assume_integer_range(imm, backend_ty, range); } pub(crate) fn codegen_rvalue_unsized( diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 3ee13b19f66..bbf87a59942 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -217,6 +217,27 @@ pub trait BuilderMethods<'a, 'tcx>: dest: PlaceRef<'tcx, Self::Value>, ); + /// Emits an `assume` that the integer value `imm` of type `ty` is contained in `range`. + /// + /// This *always* emits the assumption, so you probably want to check the + /// optimization level and `Scalar::is_always_valid` before calling it. + fn assume_integer_range(&mut self, imm: Self::Value, ty: Self::Type, range: WrappingRange) { + let WrappingRange { start, end } = range; + + // Perhaps one day we'll be able to use assume operand bundles for this, + // but for now this encoding with a single icmp+assume is best per + // <https://github.com/llvm/llvm-project/issues/123278#issuecomment-2597440158> + let shifted = if start == 0 { + imm + } else { + let low = self.const_uint_big(ty, start); + self.sub(imm, low) + }; + let width = self.const_uint_big(ty, u128::wrapping_sub(end, start)); + let cmp = self.icmp(IntPredicate::IntULE, shifted, width); + self.assume(cmp); + } + fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); fn nonnull_metadata(&mut self, load: Self::Value); diff --git a/tests/codegen/intrinsics/transmute-niched.rs b/tests/codegen/intrinsics/transmute-niched.rs index f5b7bd2efea..88119ccb8b2 100644 --- a/tests/codegen/intrinsics/transmute-niched.rs +++ b/tests/codegen/intrinsics/transmute-niched.rs @@ -17,12 +17,13 @@ pub enum SmallEnum { // CHECK-LABEL: @check_to_enum( #[no_mangle] pub unsafe fn check_to_enum(x: i8) -> SmallEnum { - // OPT: %0 = icmp uge i8 %x, 10 - // OPT: call void @llvm.assume(i1 %0) - // OPT: %1 = icmp ule i8 %x, 12 + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i8 %x, 10 + // OPT: %1 = icmp ule i8 %0, 2 // OPT: call void @llvm.assume(i1 %1) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i8 %x transmute(x) @@ -31,12 +32,13 @@ pub unsafe fn check_to_enum(x: i8) -> SmallEnum { // CHECK-LABEL: @check_from_enum( #[no_mangle] pub unsafe fn check_from_enum(x: SmallEnum) -> i8 { - // OPT: %0 = icmp uge i8 %x, 10 - // OPT: call void @llvm.assume(i1 %0) - // OPT: %1 = icmp ule i8 %x, 12 + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i8 %x, 10 + // OPT: %1 = icmp ule i8 %0, 2 // OPT: call void @llvm.assume(i1 %1) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i8 %x transmute(x) @@ -45,12 +47,13 @@ pub unsafe fn check_from_enum(x: SmallEnum) -> i8 { // CHECK-LABEL: @check_to_ordering( #[no_mangle] pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering { - // OPT: %0 = icmp uge i8 %x, -1 - // OPT: %1 = icmp ule i8 %x, 1 - // OPT: %2 = or i1 %0, %1 - // OPT: call void @llvm.assume(i1 %2) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i8 %x, -1 + // OPT: %1 = icmp ule i8 %0, 2 + // OPT: call void @llvm.assume(i1 %1) + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i8 %x transmute(x) @@ -59,12 +62,13 @@ pub unsafe fn check_to_ordering(x: u8) -> std::cmp::Ordering { // CHECK-LABEL: @check_from_ordering( #[no_mangle] pub unsafe fn check_from_ordering(x: std::cmp::Ordering) -> u8 { - // OPT: %0 = icmp uge i8 %x, -1 - // OPT: %1 = icmp ule i8 %x, 1 - // OPT: %2 = or i1 %0, %1 - // OPT: call void @llvm.assume(i1 %2) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i8 %x, -1 + // OPT: %1 = icmp ule i8 %0, 2 + // OPT: call void @llvm.assume(i1 %1) + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i8 %x transmute(x) @@ -98,14 +102,15 @@ pub enum Minus100ToPlus100 { // CHECK-LABEL: @check_enum_from_char( #[no_mangle] pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 { + // CHECK-NOT: icmp + // CHECK-NOT: assume // OPT: %0 = icmp ule i32 %x, 1114111 // OPT: call void @llvm.assume(i1 %0) - // OPT: %1 = icmp uge i32 %x, -100 - // OPT: %2 = icmp ule i32 %x, 100 - // OPT: %3 = or i1 %1, %2 - // OPT: call void @llvm.assume(i1 %3) - // DBG-NOT: icmp - // DBG-NOT: assume + // OPT: %1 = sub i32 %x, -100 + // OPT: %2 = icmp ule i32 %1, 200 + // OPT: call void @llvm.assume(i1 %2) + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i32 %x transmute(x) @@ -114,14 +119,15 @@ pub unsafe fn check_enum_from_char(x: char) -> Minus100ToPlus100 { // CHECK-LABEL: @check_enum_to_char( #[no_mangle] pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char { - // OPT: %0 = icmp uge i32 %x, -100 - // OPT: %1 = icmp ule i32 %x, 100 - // OPT: %2 = or i1 %0, %1 + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i32 %x, -100 + // OPT: %1 = icmp ule i32 %0, 200 + // OPT: call void @llvm.assume(i1 %1) + // OPT: %2 = icmp ule i32 %x, 1114111 // OPT: call void @llvm.assume(i1 %2) - // OPT: %3 = icmp ule i32 %x, 1114111 - // OPT: call void @llvm.assume(i1 %3) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i32 %x transmute(x) @@ -130,16 +136,20 @@ pub unsafe fn check_enum_to_char(x: Minus100ToPlus100) -> char { // CHECK-LABEL: @check_swap_pair( #[no_mangle] pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) { + // CHECK-NOT: icmp + // CHECK-NOT: assume // OPT: %0 = icmp ule i32 %x.0, 1114111 // OPT: call void @llvm.assume(i1 %0) - // OPT: %1 = icmp uge i32 %x.0, 1 - // OPT: call void @llvm.assume(i1 %1) - // OPT: %2 = icmp uge i32 %x.1, 1 + // OPT: %1 = sub i32 %x.0, 1 + // OPT: %2 = icmp ule i32 %1, -2 // OPT: call void @llvm.assume(i1 %2) - // OPT: %3 = icmp ule i32 %x.1, 1114111 - // OPT: call void @llvm.assume(i1 %3) - // DBG-NOT: icmp - // DBG-NOT: assume + // OPT: %3 = sub i32 %x.1, 1 + // OPT: %4 = icmp ule i32 %3, -2 + // OPT: call void @llvm.assume(i1 %4) + // OPT: %5 = icmp ule i32 %x.1, 1114111 + // OPT: call void @llvm.assume(i1 %5) + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: %[[P1:.+]] = insertvalue { i32, i32 } poison, i32 %x.0, 0 // CHECK: %[[P2:.+]] = insertvalue { i32, i32 } %[[P1]], i32 %x.1, 1 // CHECK: ret { i32, i32 } %[[P2]] @@ -150,14 +160,15 @@ pub unsafe fn check_swap_pair(x: (char, NonZero<u32>)) -> (NonZero<u32>, char) { // CHECK-LABEL: @check_bool_from_ordering( #[no_mangle] pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool { - // OPT: %0 = icmp uge i8 %x, -1 - // OPT: %1 = icmp ule i8 %x, 1 - // OPT: %2 = or i1 %0, %1 + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = sub i8 %x, -1 + // OPT: %1 = icmp ule i8 %0, 2 + // OPT: call void @llvm.assume(i1 %1) + // OPT: %2 = icmp ule i8 %x, 1 // OPT: call void @llvm.assume(i1 %2) - // OPT: %3 = icmp ule i8 %x, 1 - // OPT: call void @llvm.assume(i1 %3) - // DBG-NOT: icmp - // DBG-NOT: assume + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: %[[R:.+]] = trunc i8 %x to i1 // CHECK: ret i1 %[[R]] @@ -168,14 +179,15 @@ pub unsafe fn check_bool_from_ordering(x: std::cmp::Ordering) -> bool { #[no_mangle] pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering { // CHECK: %_0 = zext i1 %x to i8 + // CHECK-NOT: icmp + // CHECK-NOT: assume // OPT: %0 = icmp ule i8 %_0, 1 // OPT: call void @llvm.assume(i1 %0) - // OPT: %1 = icmp uge i8 %_0, -1 - // OPT: %2 = icmp ule i8 %_0, 1 - // OPT: %3 = or i1 %1, %2 - // OPT: call void @llvm.assume(i1 %3) - // DBG-NOT: icmp - // DBG-NOT: assume + // OPT: %1 = sub i8 %_0, -1 + // OPT: %2 = icmp ule i8 %1, 2 + // OPT: call void @llvm.assume(i1 %2) + // CHECK-NOT: icmp + // CHECK-NOT: assume // CHECK: ret i8 %_0 transmute(x) diff --git a/tests/codegen/transmute-optimized.rs b/tests/codegen/transmute-optimized.rs index 11bd0523788..de54eecf0c0 100644 --- a/tests/codegen/transmute-optimized.rs +++ b/tests/codegen/transmute-optimized.rs @@ -110,3 +110,11 @@ pub fn char_is_negative(c: char) -> bool { let x: i32 = unsafe { std::mem::transmute(c) }; x < 0 } + +// CHECK-LABEL: i1 @transmute_to_char_is_negative(i32 +#[no_mangle] +pub fn transmute_to_char_is_negative(x: i32) -> bool { + // CHECK: ret i1 false + let _c: char = unsafe { std::mem::transmute(x) }; + x < 0 +} |