Rollup merge of #101339 - the8472:ci-randomize-debug, r=Mark-Simulacrum

enable -Zrandomize-layout in debug CI builds

This builds rustc/libs/tools with `-Zrandomize-layout` on *-debug CI runners.

Only a handful of tests and asserts break with that enabled, which is promising. One test was fixable, the rest is dealt with by disabling them through new cargo features or compiletest directives.

The config.toml flag `rust.randomize-layout` defaults to false, so it has to be explicitly enabled for now.

9 files changed, 47 insertions, 9 deletions

diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml
index 5008069542f..a2fc9d5c408 100644
--- a/compiler/rustc/Cargo.toml
+++ b/compiler/rustc/Cargo.toml
@@ -30,5 +30,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 5160b4ed0a2..7432768be4a 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -968,8 +968,8 @@ fn univariant<
     let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
     let mut max_repr_align = repr.align;
     let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-    let optimize = !repr.inhibit_struct_field_reordering();
-    if optimize && fields.len() > 1 {
+    let optimize_field_order = !repr.inhibit_struct_field_reordering();
+    if optimize_field_order && fields.len() > 1 {
         let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
         let optimizing = &mut inverse_memory_index.raw[..end];
         let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
     // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
     // Field 5 would be the first element, so memory_index is i:
     // Note: if we didn't optimize, it's already right.
-    let memory_index = if optimize {
+    let memory_index = if optimize_field_order {
         inverse_memory_index.invert_bijective_mapping()
     } else {
         debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
     }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
+
+    let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
     // Try to make this a Scalar/ScalarPair.
     if sized && size.bytes() > 0 {
         // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
                     match field.abi {
                         // For plain scalars, or vectors of them, we can't unpack
                         // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                        Abi::Scalar(_) | Abi::Vector { .. } if optimize_abi => {
                             abi = field.abi;
                         }
                         // But scalar pairs are Rust-specific and get
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index df29b3d54f0..be42bc84932 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -43,14 +43,17 @@ bitflags! {
         const IS_SIMD            = 1 << 1;
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
+        // On its own it does not prevent ABI optimizations.
         const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.field_shuffle_seed`
+        // If true, the type's crate has opted into layout randomization.
+        // Other flags can still inhibit reordering and thus randomization.
+        // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
+        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_LINEAR.bits();
+        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
         self.c() || self.int.is_some()
     }
 
+    pub fn inhibit_newtype_abi_optimization(&self) -> bool {
+        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+    }
+
     /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
     /// e.g. `repr(C)` or `repr(<int>)`.
     pub fn inhibit_struct_field_reordering(&self) -> bool {
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
     }
 
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index 91cbffcd707..6d6d3f35a4b 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -23,6 +23,7 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_hir_typeck = { path = "../rustc_hir_typeck" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
 rustc_lint = { path = "../rustc_lint" }
@@ -72,6 +73,10 @@ ctrlc = "3.4.4"
 # tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
+rustc_randomized_layouts = [
+    'rustc_index/rustc_randomized_layouts',
+    'rustc_middle/rustc_randomized_layouts'
+]
 rustc_use_parallel_compiler = [
     'rustc_data_structures/rustc_use_parallel_compiler',
     'rustc_interface/rustc_use_parallel_compiler',
diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml
index 92ea3f278dc..f7d18f84e34 100644
--- a/compiler/rustc_index/Cargo.toml
+++ b/compiler/rustc_index/Cargo.toml
@@ -20,4 +20,5 @@ nightly = [
     "dep:rustc_macros",
     "rustc_index_macros/nightly",
 ]
+rustc_randomized_layouts = []
 # tidy-alphabetical-end
diff --git a/compiler/rustc_index/src/lib.rs b/compiler/rustc_index/src/lib.rs
index f773b5b46ad..52f354b8eca 100644
--- a/compiler/rustc_index/src/lib.rs
+++ b/compiler/rustc_index/src/lib.rs
@@ -33,8 +33,19 @@ pub use vec::IndexVec;
 ///
 /// </div>
 #[macro_export]
+#[cfg(not(feature = "rustc_randomized_layouts"))]
 macro_rules! static_assert_size {
     ($ty:ty, $size:expr) => {
         const _: [(); $size] = [(); ::std::mem::size_of::<$ty>()];
     };
 }
+
+#[macro_export]
+#[cfg(feature = "rustc_randomized_layouts")]
+macro_rules! static_assert_size {
+    ($ty:ty, $size:expr) => {
+        // no effect other than using the statements.
+        // struct sizes are not deterministic under randomized layouts
+        const _: (usize, usize) = ($size, ::std::mem::size_of::<$ty>());
+    };
+}
diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index 69e3b703cce..b23589afb58 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -40,5 +40,6 @@ tracing = "0.1"
 
 [features]
 # tidy-alphabetical-start
+rustc_randomized_layouts = []
 rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end
diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs
index c9bd702cce3..0320a91d142 100644
--- a/compiler/rustc_middle/src/query/plumbing.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -337,6 +337,7 @@ macro_rules! define_callbacks {
                 // Ensure that values grow no larger than 64 bytes by accident.
                 // Increase this limit if necessary, but do try to keep the size low if possible
                 #[cfg(target_pointer_width = "64")]
+                #[cfg(not(feature = "rustc_randomized_layouts"))]
                 const _: () = {
                     if mem::size_of::<Value<'static>>() > 64 {
                         panic!("{}", concat!(
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index cd94c0afad0..ee70a6346d9 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
     extension, Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
+        // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+        // always at offset zero. On the other hand we want scalar abi optimizations.
+        let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);
+
         // This is here instead of layout because the choice must make it into metadata.
-        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+        if is_box
+            || !self
+                .consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did)))
+        {
             flags.insert(ReprFlags::IS_LINEAR);
         }