Use niche-filling optimization even when multiple variants have data.

Fixes #46213

3 files changed, 102 insertions, 22 deletions

diff --git a/src/test/ui/stats/hir-stats.rs b/src/test/ui/stats/hir-stats.rs
index a24b3ada57e..5102574d4be 100644
--- a/src/test/ui/stats/hir-stats.rs
+++ b/src/test/ui/stats/hir-stats.rs
@@ -1,6 +1,7 @@
 // check-pass
 // compile-flags: -Zhir-stats
 // only-x86_64
+// ignore-stage1
 
 // The aim here is to include at least one of every different type of top-level
 // AST/HIR node reported by `-Zhir-stats`.
diff --git a/src/test/ui/stats/hir-stats.stderr b/src/test/ui/stats/hir-stats.stderr
index 0a736f7be83..c8ceb6ff22d 100644
--- a/src/test/ui/stats/hir-stats.stderr
+++ b/src/test/ui/stats/hir-stats.stderr
@@ -21,39 +21,39 @@ ast-stats-1 - MacCall                   32 ( 0.4%)             1
 ast-stats-1 - Expr                      96 ( 1.1%)             3
 ast-stats-1 Param                    160 ( 1.9%)             4            40
 ast-stats-1 FnDecl                   200 ( 2.4%)             5            40
-ast-stats-1 Variant                  240 ( 2.8%)             2           120
+ast-stats-1 Variant                  240 ( 2.9%)             2           120
 ast-stats-1 Block                    288 ( 3.4%)             6            48
 ast-stats-1 GenericBound             352 ( 4.2%)             4            88
 ast-stats-1 - Trait                    352 ( 4.2%)             4
 ast-stats-1 AssocItem                416 ( 4.9%)             4           104
 ast-stats-1 - TyAlias                  208 ( 2.5%)             2
 ast-stats-1 - Fn                       208 ( 2.5%)             2
-ast-stats-1 GenericParam             520 ( 6.1%)             5           104
-ast-stats-1 PathSegment              720 ( 8.5%)            30            24
-ast-stats-1 Expr                     832 ( 9.8%)             8           104
+ast-stats-1 GenericParam             480 ( 5.7%)             5            96
+ast-stats-1 PathSegment              720 ( 8.6%)            30            24
+ast-stats-1 Expr                     832 ( 9.9%)             8           104
 ast-stats-1 - Path                     104 ( 1.2%)             1
 ast-stats-1 - Match                    104 ( 1.2%)             1
 ast-stats-1 - Struct                   104 ( 1.2%)             1
 ast-stats-1 - Lit                      208 ( 2.5%)             2
 ast-stats-1 - Block                    312 ( 3.7%)             3
-ast-stats-1 Pat                      840 ( 9.9%)             7           120
+ast-stats-1 Pat                      840 (10.0%)             7           120
 ast-stats-1 - Struct                   120 ( 1.4%)             1
 ast-stats-1 - Wild                     120 ( 1.4%)             1
 ast-stats-1 - Ident                    600 ( 7.1%)             5
-ast-stats-1 Ty                     1_344 (15.9%)            14            96
+ast-stats-1 Ty                     1_344 (16.0%)            14            96
 ast-stats-1 - Rptr                      96 ( 1.1%)             1
 ast-stats-1 - Ptr                       96 ( 1.1%)             1
 ast-stats-1 - ImplicitSelf             192 ( 2.3%)             2
 ast-stats-1 - Path                     960 (11.4%)            10
-ast-stats-1 Item                   1_656 (19.6%)             9           184
+ast-stats-1 Item                   1_656 (19.7%)             9           184
 ast-stats-1 - Trait                    184 ( 2.2%)             1
 ast-stats-1 - Enum                     184 ( 2.2%)             1
 ast-stats-1 - ForeignMod               184 ( 2.2%)             1
 ast-stats-1 - Impl                     184 ( 2.2%)             1
 ast-stats-1 - Fn                       368 ( 4.4%)             2
-ast-stats-1 - Use                      552 ( 6.5%)             3
+ast-stats-1 - Use                      552 ( 6.6%)             3
 ast-stats-1 ----------------------------------------------------------------
-ast-stats-1 Total                  8_456
+ast-stats-1 Total                  8_416
 ast-stats-1
 ast-stats-2 POST EXPANSION AST STATS
 ast-stats-2 Name                Accumulated Size         Count     Item Size
@@ -86,12 +86,12 @@ ast-stats-2 - Trait                    352 ( 3.8%)             4
 ast-stats-2 AssocItem                416 ( 4.5%)             4           104
 ast-stats-2 - TyAlias                  208 ( 2.3%)             2
 ast-stats-2 - Fn                       208 ( 2.3%)             2
-ast-stats-2 GenericParam             520 ( 5.7%)             5           104
-ast-stats-2 PathSegment              792 ( 8.6%)            33            24
-ast-stats-2 Pat                      840 ( 9.1%)             7           120
+ast-stats-2 GenericParam             480 ( 5.2%)             5            96
+ast-stats-2 PathSegment              792 ( 8.7%)            33            24
+ast-stats-2 Pat                      840 ( 9.2%)             7           120
 ast-stats-2 - Struct                   120 ( 1.3%)             1
 ast-stats-2 - Wild                     120 ( 1.3%)             1
-ast-stats-2 - Ident                    600 ( 6.5%)             5
+ast-stats-2 - Ident                    600 ( 6.6%)             5
 ast-stats-2 Expr                     936 (10.2%)             9           104
 ast-stats-2 - Path                     104 ( 1.1%)             1
 ast-stats-2 - Match                    104 ( 1.1%)             1
@@ -99,12 +99,12 @@ ast-stats-2 - Struct                   104 ( 1.1%)             1
 ast-stats-2 - InlineAsm                104 ( 1.1%)             1
 ast-stats-2 - Lit                      208 ( 2.3%)             2
 ast-stats-2 - Block                    312 ( 3.4%)             3
-ast-stats-2 Ty                     1_344 (14.6%)            14            96
+ast-stats-2 Ty                     1_344 (14.7%)            14            96
 ast-stats-2 - Rptr                      96 ( 1.0%)             1
 ast-stats-2 - Ptr                       96 ( 1.0%)             1
 ast-stats-2 - ImplicitSelf             192 ( 2.1%)             2
 ast-stats-2 - Path                     960 (10.5%)            10
-ast-stats-2 Item                   2_024 (22.0%)            11           184
+ast-stats-2 Item                   2_024 (22.1%)            11           184
 ast-stats-2 - Trait                    184 ( 2.0%)             1
 ast-stats-2 - Enum                     184 ( 2.0%)             1
 ast-stats-2 - ExternCrate              184 ( 2.0%)             1
@@ -113,7 +113,7 @@ ast-stats-2 - Impl                     184 ( 2.0%)             1
 ast-stats-2 - Fn                       368 ( 4.0%)             2
 ast-stats-2 - Use                      736 ( 8.0%)             4
 ast-stats-2 ----------------------------------------------------------------
-ast-stats-2 Total                  9_184
+ast-stats-2 Total                  9_144
 ast-stats-2
 hir-stats HIR STATS
 hir-stats Name                Accumulated Size         Count     Item Size
@@ -121,7 +121,7 @@ hir-stats ----------------------------------------------------------------
 hir-stats ForeignItemRef            24 ( 0.2%)             1            24
 hir-stats Mod                       32 ( 0.3%)             1            32
 hir-stats ExprField                 40 ( 0.4%)             1            40
-hir-stats TraitItemRef              56 ( 0.5%)             2            28
+hir-stats TraitItemRef              56 ( 0.6%)             2            28
 hir-stats Param                     64 ( 0.6%)             2            32
 hir-stats Local                     64 ( 0.6%)             1            64
 hir-stats InlineAsm                 72 ( 0.7%)             1            72
@@ -135,11 +135,11 @@ hir-stats - Semi                      32 ( 0.3%)             1
 hir-stats - Expr                      32 ( 0.3%)             1
 hir-stats FnDecl                   120 ( 1.2%)             3            40
 hir-stats Attribute                128 ( 1.3%)             4            32
+hir-stats GenericArg               128 ( 1.3%)             4            32
+hir-stats - Type                      32 ( 0.3%)             1
+hir-stats - Lifetime                  96 ( 0.9%)             3
 hir-stats GenericArgs              144 ( 1.4%)             3            48
 hir-stats Variant                  160 ( 1.6%)             2            80
-hir-stats GenericArg               160 ( 1.6%)             4            40
-hir-stats - Type                      40 ( 0.4%)             1
-hir-stats - Lifetime                 120 ( 1.2%)             3
 hir-stats GenericBound             192 ( 1.9%)             4            48
 hir-stats - Trait                    192 ( 1.9%)             4
 hir-stats WherePredicate           216 ( 2.1%)             3            72
@@ -151,7 +151,7 @@ hir-stats - Wild                      88 ( 0.9%)             1
 hir-stats - Struct                    88 ( 0.9%)             1
 hir-stats - Binding                  264 ( 2.6%)             3
 hir-stats Generics                 560 ( 5.5%)            10            56
-hir-stats Expr                     768 ( 7.5%)            12            64
+hir-stats Expr                     768 ( 7.6%)            12            64
 hir-stats - Path                      64 ( 0.6%)             1
 hir-stats - Struct                    64 ( 0.6%)             1
 hir-stats - Match                     64 ( 0.6%)             1
@@ -173,5 +173,5 @@ hir-stats - Path                     936 ( 9.2%)            13
 hir-stats Path                   1_536 (15.1%)            32            48
 hir-stats PathSegment            2_240 (22.0%)            40            56
 hir-stats ----------------------------------------------------------------
-hir-stats Total                 10_200
+hir-stats Total                 10_168
 hir-stats
diff --git a/src/test/ui/structs-enums/type-sizes.rs b/src/test/ui/structs-enums/type-sizes.rs
index 73a11a5e743..7a23f13630a 100644
--- a/src/test/ui/structs-enums/type-sizes.rs
+++ b/src/test/ui/structs-enums/type-sizes.rs
@@ -120,6 +120,54 @@ pub enum AlwaysTaggedBecauseItHasNoNiche {
     B
 }
 
+pub enum NicheFilledMultipleFields {
+    A(bool, u8),
+    B(u8),
+    C(u8),
+    D(bool),
+    E,
+    F,
+    G,
+}
+
+struct BoolInTheMiddle(std::num::NonZeroU16, bool, u8);
+
+enum NicheWithData {
+    A,
+    B([u16; 5]),
+    Largest { a1: u32, a2: BoolInTheMiddle, a3: u32 },
+    C,
+    D(u32, u32),
+}
+
+// A type with almost 2^16 invalid values.
+#[repr(u16)]
+pub enum NicheU16 {
+    _0,
+}
+
+pub enum EnumManyVariant<X> {
+    Dataful(u8, X),
+
+    // 0x100 niche variants.
+    _00, _01, _02, _03, _04, _05, _06, _07, _08, _09, _0A, _0B, _0C, _0D, _0E, _0F,
+    _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _1A, _1B, _1C, _1D, _1E, _1F,
+    _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _2A, _2B, _2C, _2D, _2E, _2F,
+    _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _3A, _3B, _3C, _3D, _3E, _3F,
+    _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _4A, _4B, _4C, _4D, _4E, _4F,
+    _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _5A, _5B, _5C, _5D, _5E, _5F,
+    _60, _61, _62, _63, _64, _65, _66, _67, _68, _69, _6A, _6B, _6C, _6D, _6E, _6F,
+    _70, _71, _72, _73, _74, _75, _76, _77, _78, _79, _7A, _7B, _7C, _7D, _7E, _7F,
+    _80, _81, _82, _83, _84, _85, _86, _87, _88, _89, _8A, _8B, _8C, _8D, _8E, _8F,
+    _90, _91, _92, _93, _94, _95, _96, _97, _98, _99, _9A, _9B, _9C, _9D, _9E, _9F,
+    _A0, _A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8, _A9, _AA, _AB, _AC, _AD, _AE, _AF,
+    _B0, _B1, _B2, _B3, _B4, _B5, _B6, _B7, _B8, _B9, _BA, _BB, _BC, _BD, _BE, _BF,
+    _C0, _C1, _C2, _C3, _C4, _C5, _C6, _C7, _C8, _C9, _CA, _CB, _CC, _CD, _CE, _CF,
+    _D0, _D1, _D2, _D3, _D4, _D5, _D6, _D7, _D8, _D9, _DA, _DB, _DC, _DD, _DE, _DF,
+    _E0, _E1, _E2, _E3, _E4, _E5, _E6, _E7, _E8, _E9, _EA, _EB, _EC, _ED, _EE, _EF,
+    _F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF,
+}
+
 pub fn main() {
     assert_eq!(size_of::<u8>(), 1 as usize);
     assert_eq!(size_of::<u32>(), 4 as usize);
@@ -170,4 +218,35 @@ pub fn main() {
     assert_eq!(size_of::<AlwaysTaggedBecauseItHasNoNiche>(), 8);
     assert_eq!(size_of::<Option<AlwaysTaggedBecauseItHasNoNiche>>(), 8);
     assert_eq!(size_of::<Option<Option<AlwaysTaggedBecauseItHasNoNiche>>>(), 8);
+
+    assert_eq!(size_of::<NicheFilledMultipleFields>(), 2);
+    assert_eq!(size_of::<Option<NicheFilledMultipleFields>>(), 2);
+    assert_eq!(size_of::<Option<Option<NicheFilledMultipleFields>>>(), 2);
+
+    struct S1{ a: u16, b: std::num::NonZeroU16, c: u16, d: u8, e: u32, f: u64, g:[u8;2] }
+    assert_eq!(size_of::<S1>(), 24);
+    assert_eq!(size_of::<Option<S1>>(), 24);
+
+    assert_eq!(size_of::<NicheWithData>(), 12);
+    assert_eq!(size_of::<Option<NicheWithData>>(), 12);
+    assert_eq!(size_of::<Option<Option<NicheWithData>>>(), 12);
+    assert_eq!(
+        size_of::<Option<Option2<&(), Option<NicheWithData>>>>(),
+        size_of::<(&(), NicheWithData)>()
+    );
+
+    pub enum FillPadding { A(std::num::NonZeroU8, u32), B }
+    assert_eq!(size_of::<FillPadding>(), 8);
+    assert_eq!(size_of::<Option<FillPadding>>(), 8);
+    assert_eq!(size_of::<Option<Option<FillPadding>>>(), 8);
+
+    assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8), u16>>(), 4);
+    assert_eq!(size_of::<Option<Result<(std::num::NonZeroU8, u8, u8), u16>>>(), 4);
+    assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8, u8), u16>>(), 4);
+
+    assert_eq!(size_of::<EnumManyVariant<u16>>(), 6);
+    assert_eq!(size_of::<EnumManyVariant<NicheU16>>(), 4);
+    assert_eq!(size_of::<EnumManyVariant<Option<NicheU16>>>(), 4);
+    assert_eq!(size_of::<EnumManyVariant<Option2<NicheU16,u8>>>(), 6);
+    assert_eq!(size_of::<EnumManyVariant<Option<(NicheU16,u8)>>>(), 6);
 }