Auto merge of #113166 - moulins:ref-niches-initial, r=oli-obk

Prototype: Add unstable `-Z reference-niches` option

MCP: rust-lang/compiler-team#641
Relevant RFC: rust-lang/rfcs#3204

This prototype adds a new `-Z reference-niches` option, controlling the range of valid bit-patterns for reference types (`&T` and `&mut T`), thereby enabling new enum niching opportunities. Like `-Z randomize-layout`, this setting is crate-local; as such, references to built-in types (primitives, tuples, ...) are not affected.

The possible settings are (here, `MAX` denotes the all-1 bit-pattern):
| `-Z reference-niches=` | Valid range |
|:---:|:---:|
| `null` (the default) | `1..=MAX` |
| `size` | `1..=(MAX- size)` |
| `align` | `align..=MAX.align_down_to(align)` |
| `size,align` | `align..=(MAX-size).align_down_to(align)` |

------

This is very WIP, and I'm not sure the approach I've taken here is the best one, but stage 1 tests pass locally; I believe this is in a good enough state to unleash this upon unsuspecting 3rd-party code, and see what breaks.

2 files changed, 32 insertions, 0 deletions

diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index a8147ede970..1766e97b67d 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -3117,6 +3117,7 @@ pub(crate) mod dep_tracking {
     use rustc_feature::UnstableFeatures;
     use rustc_span::edition::Edition;
     use rustc_span::RealFileName;
+    use rustc_target::abi::ReferenceNichePolicy;
     use rustc_target::spec::{CodeModel, MergeFunctions, PanicStrategy, RelocModel};
     use rustc_target::spec::{
         RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TargetTriple, TlsModel,
@@ -3212,6 +3213,7 @@ pub(crate) mod dep_tracking {
         OomStrategy,
         LanguageIdentifier,
         TraitSolver,
+        ReferenceNichePolicy,
     );
 
     impl<T1, T2> DepTrackingHash for (T1, T2)
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 39efe9abeec..0c66121c72f 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -6,6 +6,7 @@ use crate::{lint, EarlyErrorHandler};
 use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::ColorConfig;
 use rustc_errors::{LanguageIdentifier, TerminalUrl};
+use rustc_target::abi::ReferenceNichePolicy;
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet};
 use rustc_target::spec::{
     RelocModel, RelroLevel, SplitDebuginfo, StackProtector, TargetTriple, TlsModel,
@@ -421,6 +422,8 @@ mod desc {
     pub const parse_proc_macro_execution_strategy: &str =
         "one of supported execution strategies (`same-thread`, or `cross-thread`)";
     pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`";
+    pub const parse_opt_reference_niches: &str =
+        "`null`, or a `,` separated combination of `size` or `align`";
 }
 
 mod parse {
@@ -1253,6 +1256,31 @@ mod parse {
         };
         true
     }
+
+    pub(crate) fn parse_opt_reference_niches(
+        slot: &mut Option<ReferenceNichePolicy>,
+        v: Option<&str>,
+    ) -> bool {
+        let Some(s) = v else {
+            return false;
+        };
+
+        let slot = slot.get_or_insert_default();
+
+        if s == "null" {
+            return true;
+        }
+
+        for opt in s.split(",") {
+            match opt {
+                "size" => slot.size = true,
+                "align" => slot.align = true,
+                _ => return false,
+            }
+        }
+
+        true
+    }
 }
 
 options! {
@@ -1701,6 +1729,8 @@ options! {
         "enable queries of the dependency graph for regression testing (default: no)"),
     randomize_layout: bool = (false, parse_bool, [TRACKED],
         "randomize the layout of types (default: no)"),
+    reference_niches: Option<ReferenceNichePolicy> = (None, parse_opt_reference_niches, [TRACKED],
+        "override the set of discriminant niches that may be exposed by references"),
     relax_elf_relocations: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "whether ELF relocations can be relaxed"),
     relro_level: Option<RelroLevel> = (None, parse_relro_level, [TRACKED],