Rollup merge of #133681 - RalfJung:niches, r=wesleywiser

improve TagEncoding::Niche docs, sanity check, and UB checks

Turns out the `niche_variants` range can actually contain the `untagged_variant`. We should report this as UB in Miri, so this PR implements that.

Also rename `partially_check_layout` to `layout_sanity_check` for better consistency with how similar functions are called in other parts of the compiler.

Turns out my adjustments to the transmutation logic also fix https://github.com/rust-lang/rust/issues/126267.

2 files changed, 42 insertions, 0 deletions

diff --git a/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.rs b/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.rs
new file mode 100644
index 00000000000..bd02e7f5fb4
--- /dev/null
+++ b/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.rs
@@ -0,0 +1,27 @@
+// Validity makes this fail at the wrong place.
+//@compile-flags: -Zmiri-disable-validation
+use std::mem;
+
+// This enum has untagged variant idx 1, with niche_variants being 0..=2
+// and niche_start being 2.
+// That means the untagged variants is in the niche variant range!
+// However, using the corresponding value (2+1 = 3) is not a valid encoding of this variant.
+#[derive(Copy, Clone, PartialEq)]
+enum Foo {
+    Var1,
+    Var2(bool),
+    Var3,
+}
+
+fn main() {
+    unsafe {
+        assert!(Foo::Var2(false) == mem::transmute(0u8));
+        assert!(Foo::Var2(true) == mem::transmute(1u8));
+        assert!(Foo::Var1 == mem::transmute(2u8));
+        assert!(Foo::Var3 == mem::transmute(4u8));
+
+        let invalid: Foo = mem::transmute(3u8);
+        assert!(matches!(invalid, Foo::Var2(_)));
+        //~^ ERROR: invalid tag
+    }
+}
diff --git a/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.stderr b/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.stderr
new file mode 100644
index 00000000000..759dbc36380
--- /dev/null
+++ b/src/tools/miri/tests/fail/enum-untagged-variant-invalid-encoding.stderr
@@ -0,0 +1,15 @@
+error: Undefined Behavior: enum value has invalid tag: 0x03
+  --> tests/fail/enum-untagged-variant-invalid-encoding.rs:LL:CC
+   |
+LL |         assert!(matches!(invalid, Foo::Var2(_)));
+   |                          ^^^^^^^ enum value has invalid tag: 0x03
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `main` at tests/fail/enum-untagged-variant-invalid-encoding.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+