don't generate partially-undef consts

6 files changed, 56 insertions, 24 deletions

diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 901fa9fb047..3ece1659b9e 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -36,11 +36,6 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
         alloc: &'a Allocation,
         range: Range<usize>,
     ) {
-        /// Allocations larger than this will only be codegen'd as entirely initialized or entirely undef.
-        /// This avoids compile time regressions when an alloc would have many chunks,
-        /// e.g. for `[(u64, u8); N]`, which has undef padding in each element.
-        const MAX_PARTIALLY_UNDEF_SIZE: usize = 1024;
-
         let mut chunks = alloc
             .init_mask()
             .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end));
@@ -57,21 +52,30 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
             }
         };
 
-        if range.len() > MAX_PARTIALLY_UNDEF_SIZE {
+        // Generating partially-uninit consts inhibits optimizations, so it is disabled by default.
+        // See https://github.com/rust-lang/rust/issues/84565.
+        let allow_partially_uninit =
+            match cx.sess().opts.debugging_opts.partially_uninit_const_threshold {
+                Some(max) => range.len() <= max,
+                None => false,
+            };
+
+        if allow_partially_uninit {
+            llvals.extend(chunks.map(chunk_to_llval));
+        } else {
             let llval = match (chunks.next(), chunks.next()) {
                 (Some(chunk), None) => {
                     // exactly one chunk, either fully init or fully uninit
                     chunk_to_llval(chunk)
                 }
                 _ => {
-                    // partially uninit
+                    // partially uninit, codegen as if it was initialized
+                    // (using some arbitrary value for uninit bytes)
                     let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
                     cx.const_bytes(bytes)
                 }
             };
             llvals.push(llval);
-        } else {
-            llvals.extend(chunks.map(chunk_to_llval));
         }
     }
 
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index b8961434006..afab919bc3c 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -743,6 +743,7 @@ fn test_debugging_options_tracking_hash() {
     tracked!(no_profiler_runtime, true);
     tracked!(osx_rpath_install_name, true);
     tracked!(panic_abort_tests, true);
+    tracked!(partially_uninit_const_threshold, Some(123));
     tracked!(plt, Some(true));
     tracked!(polonius, true);
     tracked!(precise_enum_drop_elaboration, false);
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index f36fc29e974..9a1be40558c 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1186,6 +1186,9 @@ options! {
         "support compiling tests with panic=abort (default: no)"),
     parse_only: bool = (false, parse_bool, [UNTRACKED],
         "parse only; do not compile, assemble, or link (default: no)"),
+    partially_uninit_const_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
+        "allow generating const initializers with mixed init/uninit bytes, \
+        and set the maximum total size of a const allocation for which this is allowed (default: never)"),
     perf_stats: bool = (false, parse_bool, [UNTRACKED],
         "print some performance-related statistics (default: no)"),
     plt: Option<bool> = (None, parse_opt_bool, [TRACKED],
diff --git a/src/test/codegen/consts.rs b/src/test/codegen/consts.rs
index e47a9f9ee20..7f945299c22 100644
--- a/src/test/codegen/consts.rs
+++ b/src/test/codegen/consts.rs
@@ -43,7 +43,7 @@ pub fn inline_enum_const() -> E<i8, i16> {
 #[no_mangle]
 pub fn low_align_const() -> E<i16, [i16; 3]> {
     // Check that low_align_const and high_align_const use the same constant
-    // CHECK: memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 2 %1, i8* align 2 getelementptr inbounds (<{ [4 x i8], [4 x i8] }>, <{ [4 x i8], [4 x i8] }>* [[LOW_HIGH]], i32 0, i32 0, i32 0), i{{(32|64)}} 8, i1 false)
+    // CHECK: memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 2 %1, i8* align 2 getelementptr inbounds (<{ [8 x i8] }>, <{ [8 x i8] }>* [[LOW_HIGH]], i32 0, i32 0, i32 0), i{{(32|64)}} 8, i1 false)
     *&E::A(0)
 }
 
@@ -51,6 +51,6 @@ pub fn low_align_const() -> E<i16, [i16; 3]> {
 #[no_mangle]
 pub fn high_align_const() -> E<i16, i32> {
     // Check that low_align_const and high_align_const use the same constant
-    // CHECK: memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [4 x i8], [4 x i8] }>, <{ [4 x i8], [4 x i8] }>* [[LOW_HIGH]], i32 0, i32 0, i32 0), i{{(32|64)}} 8, i1 false)
+    // CHECK: memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [8 x i8] }>, <{ [8 x i8] }>* [[LOW_HIGH]], i32 0, i32 0, i32 0), i{{(32|64)}} 8, i1 false)
     *&E::A(0)
 }
diff --git a/src/test/codegen/uninit-consts-allow-partially-uninit.rs b/src/test/codegen/uninit-consts-allow-partially-uninit.rs
new file mode 100644
index 00000000000..f7420e4126e
--- /dev/null
+++ b/src/test/codegen/uninit-consts-allow-partially-uninit.rs
@@ -0,0 +1,35 @@
+// compile-flags: -C no-prepopulate-passes -Z partially_uninit_const_threshold=1024
+
+// Like uninit-consts.rs, but tests that we correctly generate partially-uninit consts
+// when the (disabled by default) partially_uninit_const_threshold flag is used.
+
+#![crate_type = "lib"]
+
+use std::mem::MaybeUninit;
+
+pub struct PartiallyUninit {
+    x: u32,
+    y: MaybeUninit<[u8; 10]>
+}
+
+// This should be partially undef.
+// CHECK: [[PARTIALLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [4 x i8], [12 x i8] }> <{ [4 x i8] c"\EF\BE\AD\DE", [12 x i8] undef }>, align 4
+
+// This shouldn't contain undef, since it's larger than the 1024 byte limit.
+// CHECK: [[UNINIT_PADDING_HUGE:@[0-9]+]] = private unnamed_addr constant <{ [32768 x i8] }> <{ [32768 x i8] c"{{.+}}" }>, align 4
+
+// CHECK-LABEL: @partially_uninit
+#[no_mangle]
+pub const fn partially_uninit() -> PartiallyUninit {
+    const X: PartiallyUninit = PartiallyUninit { x: 0xdeadbeef, y: MaybeUninit::uninit() };
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [4 x i8], [12 x i8] }>, <{ [4 x i8], [12 x i8] }>* [[PARTIALLY_UNINIT]], i32 0, i32 0, i32 0), i{{(32|64)}} 16, i1 false)
+    X
+}
+
+// CHECK-LABEL: @uninit_padding_huge
+#[no_mangle]
+pub const fn uninit_padding_huge() -> [(u32, u8); 4096] {
+    const X: [(u32, u8); 4096] = [(123, 45); 4096];
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [32768 x i8] }>, <{ [32768 x i8] }>* [[UNINIT_PADDING_HUGE]], i32 0, i32 0, i32 0), i{{(32|64)}} 32768, i1 false)
+    X
+}
diff --git a/src/test/codegen/uninit-consts.rs b/src/test/codegen/uninit-consts.rs
index 518bf7f451f..c4c21e03f16 100644
--- a/src/test/codegen/uninit-consts.rs
+++ b/src/test/codegen/uninit-consts.rs
@@ -12,12 +12,9 @@ pub struct PartiallyUninit {
 }
 
 // CHECK: [[FULLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [10 x i8] }> undef
-// CHECK: [[PARTIALLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [4 x i8], [12 x i8] }> <{ [4 x i8] c"\EF\BE\AD\DE", [12 x i8] undef }>, align 4
+// CHECK: [[PARTIALLY_UNINIT:@[0-9]+]] = private unnamed_addr constant <{ [16 x i8] }> <{ [16 x i8] c"\EF\BE\AD\DE\00\00\00\00\00\00\00\00\00\00\00\00" }>, align 4
 // CHECK: [[FULLY_UNINIT_HUGE:@[0-9]+]] = private unnamed_addr constant <{ [16384 x i8] }> undef
 
-// This shouldn't contain undef, since generating huge partially undef constants is expensive.
-// CHECK: [[UNINIT_PADDING_HUGE:@[0-9]+]] = private unnamed_addr constant <{ [32768 x i8] }> <{ [32768 x i8] c"{{.+}}" }>, align 4
-
 // CHECK-LABEL: @fully_uninit
 #[no_mangle]
 pub const fn fully_uninit() -> MaybeUninit<[u8; 10]> {
@@ -30,7 +27,7 @@ pub const fn fully_uninit() -> MaybeUninit<[u8; 10]> {
 #[no_mangle]
 pub const fn partially_uninit() -> PartiallyUninit {
     const X: PartiallyUninit = PartiallyUninit { x: 0xdeadbeef, y: MaybeUninit::uninit() };
-    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [4 x i8], [12 x i8] }>, <{ [4 x i8], [12 x i8] }>* [[PARTIALLY_UNINIT]], i32 0, i32 0, i32 0), i{{(32|64)}} 16, i1 false)
+    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [16 x i8] }>, <{ [16 x i8] }>* [[PARTIALLY_UNINIT]], i32 0, i32 0, i32 0), i{{(32|64)}} 16, i1 false)
     X
 }
 
@@ -41,11 +38,3 @@ pub const fn fully_uninit_huge() -> MaybeUninit<[u32; 4096]> {
     // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [16384 x i8] }>, <{ [16384 x i8] }>* [[FULLY_UNINIT_HUGE]], i32 0, i32 0, i32 0), i{{(32|64)}} 16384, i1 false)
     F
 }
-
-// CHECK-LABEL: @uninit_padding_huge
-#[no_mangle]
-pub const fn uninit_padding_huge() -> [(u32, u8); 4096] {
-    const X: [(u32, u8); 4096] = [(123, 45); 4096];
-    // CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{(32|64)}}(i8* align 4 %1, i8* align 4 getelementptr inbounds (<{ [32768 x i8] }>, <{ [32768 x i8] }>* [[UNINIT_PADDING_HUGE]], i32 0, i32 0, i32 0), i{{(32|64)}} 32768, i1 false)
-    X
-}