From 1f06a6a25238e3f13c78a7ec2ac25af07d44a710 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 28 Mar 2025 20:50:28 -0700 Subject: Tell LLVM about impossible niche tags --- compiler/rustc_codegen_ssa/src/mir/operand.rs | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 7e355b6406a..fca0695f296 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -9,6 +9,7 @@ use rustc_middle::mir::{self, ConstValue}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::{bug, span_bug}; +use rustc_session::config::OptLevel; use tracing::{debug, instrument}; use super::place::{PlaceRef, PlaceValue}; @@ -496,6 +497,18 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)), }; + // Layout ensures that we only get here for cases where the discriminant + // value and the variant index match, since that's all `Niche` can encode. + // But for emphasis and debugging, let's double-check one anyway. + debug_assert_eq!( + self.layout + .ty + .discriminant_for_variant(bx.tcx(), untagged_variant) + .unwrap() + .val, + u128::from(untagged_variant.as_u32()), + ); + let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); // We have a subrange `niche_start..=niche_end` inside `range`. @@ -537,6 +550,21 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { relative_discr, bx.cx().const_uint(tag_llty, relative_max as u64), ); + + // Thanks to parameter attributes and load metadata, LLVM already knows + // the general valid range of the tag. It's possible, though, for there + // to be an impossible value *in the middle*, which those ranges don't + // communicate, so it's worth an `assume` to let the optimizer know. + if niche_variants.contains(&untagged_variant) + && bx.cx().sess().opts.optimize != OptLevel::No + { + let impossible = + u64::from(untagged_variant.as_u32() - niche_variants.start().as_u32()); + let impossible = bx.cx().const_uint(tag_llty, impossible); + let ne = bx.icmp(IntPredicate::IntNE, relative_discr, impossible); + bx.assume(ne); + } + (is_niche, cast_tag, niche_variants.start().as_u32() as u128) }; -- cgit 1.4.1-3-g733a5 From 502f7f9c241b35d65f4538d28c43e6feca474df2 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Mon, 7 Apr 2025 18:12:06 -0700 Subject: Address PR feedback --- compiler/rustc_codegen_ssa/src/mir/operand.rs | 4 +++- tests/codegen/enum/enum-two-variants-match.rs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'compiler/rustc_codegen_ssa/src') diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index fca0695f296..eade9e52de9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -581,7 +581,9 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { ); // In principle we could insert assumes on the possible range of `discr`, but - // currently in LLVM this seems to be a pessimization. + // currently in LLVM this isn't worth it because the original `tag` will + // have either a `range` parameter attribute or `!range` metadata, + // or come from a `transmute` that already `assume`d it. discr } diff --git a/tests/codegen/enum/enum-two-variants-match.rs b/tests/codegen/enum/enum-two-variants-match.rs index 5b6c0e2f8ad..12d9edc4d62 100644 --- a/tests/codegen/enum/enum-two-variants-match.rs +++ b/tests/codegen/enum/enum-two-variants-match.rs @@ -3,6 +3,10 @@ #![crate_type = "lib"] +// This directly tests what we emit for these matches, rather than what happens +// after optimization, so it doesn't need to worry about extra flags on the +// instructions and is less susceptible to being broken on LLVM updates. + // CHECK-LABEL: @option_match #[no_mangle] pub fn option_match(x: Option) -> u16 { @@ -103,10 +107,22 @@ pub fn option_ordering_match(x: Option) -> char { // CHECK-LABEL: @option_nonzero_match( #[no_mangle] pub fn option_nonzero_match(x: Option>) -> u16 { + // CHECK: %[[OUT:.+]] = alloca [2 x i8] + // CHECK: %[[IS_NONE:.+]] = icmp eq i16 %x, 0 // CHECK: %[[OPT_DISCR:.+]] = select i1 %[[IS_NONE]], i64 0, i64 1 // CHECK: %[[OPT_DISCR_T:.+]] = trunc nuw i64 %[[OPT_DISCR]] to i1 // CHECK: br i1 %[[OPT_DISCR_T]], label %[[BB_SOME:.+]], label %[[BB_NONE:.+]] + + // CHECK: [[BB_SOME]]: + // CHECK: store i16 987, ptr %[[OUT]] + + // CHECK: [[BB_NONE]]: + // CHECK: store i16 123, ptr %[[OUT]] + + // CHECK: %[[RET:.+]] = load i16, ptr %[[OUT]] + // CHECK: ret i16 %[[RET]] + match x { None => 123, Some(_) => 987, -- cgit 1.4.1-3-g733a5