Use niche-filling optimization even when multiple variants have data.

Fixes #46213

13 files changed, 339 insertions, 159 deletions

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index f25fdc942b0..e38572f609b 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -3075,7 +3075,8 @@ mod size_asserts {
     static_assert_size!(Block, 48);
     static_assert_size!(Expr, 104);
     static_assert_size!(ExprKind, 72);
-    static_assert_size!(Fn, 192);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Fn, 184);
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index efa33e18510..ba041810bd1 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -780,13 +780,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 }
 
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
 mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // These are in alphabetical order, which is easy to maintain.
-    static_assert_size!(Immediate, 56);
-    static_assert_size!(ImmTy<'_>, 72);
-    static_assert_size!(Operand, 64);
-    static_assert_size!(OpTy<'_>, 88);
+    static_assert_size!(Immediate, 48);
+    static_assert_size!(ImmTy<'_>, 64);
+    static_assert_size!(Operand, 56);
+    static_assert_size!(OpTy<'_>, 80);
 }
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 81b0b5a7459..b328892906d 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -890,6 +890,8 @@ mod size_asserts {
     static_assert_size!(MemPlaceMeta, 24);
     static_assert_size!(MemPlace, 40);
     static_assert_size!(MPlaceTy<'_>, 64);
-    static_assert_size!(Place, 48);
-    static_assert_size!(PlaceTy<'_>, 72);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Place, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(PlaceTy<'_>, 64);
 }
diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
index 07a96dd7dbb..e351b650a16 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
@@ -117,6 +117,10 @@ pub trait ObligationProcessor {
 }
 
 /// The result type used by `process_obligation`.
+// `repr(C)` to inhibit the niche filling optimization. Otherwise, the `match` appearing
+// in `process_obligations` is significantly slower, which can substantially affect
+// benchmarks like `rustc-perf`'s inflate and keccak.
+#[repr(C)]
 #[derive(Debug)]
 pub enum ProcessResult<O, E> {
     Unchanged,
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 37ff6dcff7d..513225e1606 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -69,8 +69,8 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a, ErrorGuaranteed>>;
 // (See also the comment on `DiagnosticBuilder`'s `diagnostic` field.)
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(PResult<'_, ()>, 16);
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(PResult<'_, bool>, 24);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
+rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16);
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)]
 pub enum SuggestionStyle {
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 50e7c5d2d04..a668c0e95ce 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -3473,12 +3473,15 @@ mod size_asserts {
     static_assert_size!(FnDecl<'_>, 40);
     static_assert_size!(ForeignItem<'_>, 72);
     static_assert_size!(ForeignItemKind<'_>, 40);
-    static_assert_size!(GenericArg<'_>, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(GenericArg<'_>, 32);
     static_assert_size!(GenericBound<'_>, 48);
     static_assert_size!(Generics<'_>, 56);
     static_assert_size!(Impl<'_>, 80);
-    static_assert_size!(ImplItem<'_>, 88);
-    static_assert_size!(ImplItemKind<'_>, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(ImplItem<'_>, 80);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(ImplItemKind<'_>, 32);
     static_assert_size!(Item<'_>, 80);
     static_assert_size!(ItemKind<'_>, 48);
     static_assert_size!(Local<'_>, 64);
@@ -3490,8 +3493,10 @@ mod size_asserts {
     static_assert_size!(QPath<'_>, 24);
     static_assert_size!(Stmt<'_>, 32);
     static_assert_size!(StmtKind<'_>, 16);
-    static_assert_size!(TraitItem<'_>, 96);
-    static_assert_size!(TraitItemKind<'_>, 56);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(TraitItem<'static>, 88);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(TraitItemKind<'_>, 48);
     static_assert_size!(Ty<'_>, 72);
     static_assert_size!(TyKind<'_>, 56);
 }
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index bf63b8efaf7..e149535bec7 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -1231,7 +1231,8 @@ pub enum BinOp {
 mod size_asserts {
     use super::*;
     // These are in alphabetical order, which is easy to maintain.
-    static_assert_size!(AggregateKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(AggregateKind<'_>, 40);
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 7e543929b0f..c50f8b0eebe 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -825,8 +825,12 @@ mod size_asserts {
     static_assert_size!(Block, 56);
     static_assert_size!(Expr<'_>, 64);
     static_assert_size!(ExprKind<'_>, 40);
-    static_assert_size!(Pat<'_>, 72);
-    static_assert_size!(PatKind<'_>, 56);
-    static_assert_size!(Stmt<'_>, 56);
-    static_assert_size!(StmtKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Pat<'_>, 64);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(PatKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Stmt<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(StmtKind<'_>, 40);
 }
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 0a5463a021f..abb7ddd88b1 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -22,7 +22,7 @@ use rustc_target::abi::call::{
 use rustc_target::abi::*;
 use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target};
 
-use std::cmp;
+use std::cmp::{self, Ordering};
 use std::fmt;
 use std::iter;
 use std::num::NonZeroUsize;
@@ -1046,131 +1046,191 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 // that allow representation optimization.)
                 assert!(def.is_enum());
 
-                // The current code for niche-filling relies on variant indices
-                // instead of actual discriminants, so untagged enums with
-                // explicit discriminants (RFC #2363) would misbehave.
-                let no_explicit_discriminants = def
-                    .variants()
-                    .iter_enumerated()
-                    .all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i.as_u32()));
-
-                let mut niche_filling_layout = None;
-
-                // Niche-filling enum optimization.
-                if !def.repr().inhibit_enum_layout_opt() && no_explicit_discriminants {
-                    let mut untagged_variant = None;
-                    let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
+                // Until we've decided whether to use the tagged or
+                // niche filling LayoutS, we don't want to intern the
+                // variant layouts, so we can't store them in the
+                // overall LayoutS. Store the overall LayoutS
+                // and the variant LayoutSs here until then.
+                struct TmpLayout<'tcx> {
+                    layout: LayoutS<'tcx>,
+                    variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
+                }
 
-                    // Find one non-ZST variant.
-                    'variants: for (v, fields) in variants.iter_enumerated() {
-                        if absent(fields) {
-                            continue 'variants;
+                let calculate_niche_filling_layout =
+                    || -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
+                        // The current code for niche-filling relies on variant indices
+                        // instead of actual discriminants, so enums with
+                        // explicit discriminants (RFC #2363) would misbehave.
+                        if def.repr().inhibit_enum_layout_opt()
+                            || def
+                                .variants()
+                                .iter_enumerated()
+                                .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
+                        {
+                            return Ok(None);
                         }
-                        for f in fields {
-                            if !f.is_zst() {
-                                if untagged_variant.is_none() {
-                                    untagged_variant = Some(v);
-                                    continue 'variants;
-                                } else {
-                                    untagged_variant = None;
-                                    break 'variants;
-                                }
-                            }
+
+                        if variants.len() < 2 {
+                            return Ok(None);
                         }
-                        niche_variants = *niche_variants.start().min(&v)..=v;
-                    }
 
-                    if niche_variants.start() > niche_variants.end() {
-                        untagged_variant = None;
-                    }
+                        let mut align = dl.aggregate_align;
+                        let mut variant_layouts = variants
+                            .iter_enumerated()
+                            .map(|(j, v)| {
+                                let mut st = self.univariant_uninterned(
+                                    ty,
+                                    v,
+                                    &def.repr(),
+                                    StructKind::AlwaysSized,
+                                )?;
+                                st.variants = Variants::Single { index: j };
+
+                                align = align.max(st.align);
+
+                                Ok(st)
+                            })
+                            .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+
+                        let largest_variant_index = match variant_layouts
+                            .iter_enumerated()
+                            .max_by_key(|(_i, layout)| layout.size.bytes())
+                            .map(|(i, _layout)| i)
+                        {
+                            None => return Ok(None),
+                            Some(i) => i,
+                        };
 
-                    if let Some(i) = untagged_variant {
-                        let count = (niche_variants.end().as_u32()
-                            - niche_variants.start().as_u32()
-                            + 1) as u128;
+                        let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
+                        let needs_disc = |index: VariantIdx| {
+                            index != largest_variant_index && !absent(&variants[index])
+                        };
+                        let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
+                            ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
+
+                        let count = niche_variants.size_hint().1.unwrap() as u128;
 
                         // Find the field with the largest niche
-                        let niche_candidate = variants[i]
+                        let (field_index, niche, (niche_start, niche_scalar)) = match variants
+                            [largest_variant_index]
                             .iter()
                             .enumerate()
                             .filter_map(|(j, field)| Some((j, field.largest_niche?)))
-                            .max_by_key(|(_, niche)| niche.available(dl));
-
-                        if let Some((field_index, niche, (niche_start, niche_scalar))) =
-                            niche_candidate.and_then(|(field_index, niche)| {
-                                Some((field_index, niche, niche.reserve(self, count)?))
-                            })
+                            .max_by_key(|(_, niche)| niche.available(dl))
+                            .and_then(|(j, niche)| Some((j, niche, niche.reserve(self, count)?)))
                         {
-                            let mut align = dl.aggregate_align;
-                            let st = variants
-                                .iter_enumerated()
-                                .map(|(j, v)| {
-                                    let mut st = self.univariant_uninterned(
-                                        ty,
-                                        v,
-                                        &def.repr(),
-                                        StructKind::AlwaysSized,
-                                    )?;
-                                    st.variants = Variants::Single { index: j };
+                            None => return Ok(None),
+                            Some(x) => x,
+                        };
 
-                                    align = align.max(st.align);
+                        let niche_offset = niche.offset
+                            + variant_layouts[largest_variant_index].fields.offset(field_index);
+                        let niche_size = niche.value.size(dl);
+                        let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
 
-                                    Ok(tcx.intern_layout(st))
-                                })
-                                .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+                        let all_variants_fit =
+                            variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
+                                if i == largest_variant_index {
+                                    return true;
+                                }
 
-                            let offset = st[i].fields().offset(field_index) + niche.offset;
+                                layout.largest_niche = None;
 
-                            // Align the total size to the largest alignment.
-                            let size = st[i].size().align_to(align.abi);
+                                if layout.size <= niche_offset {
+                                    // This variant will fit before the niche.
+                                    return true;
+                                }
 
-                            let abi = if st.iter().all(|v| v.abi().is_uninhabited()) {
-                                Abi::Uninhabited
-                            } else if align == st[i].align() && size == st[i].size() {
-                                // When the total alignment and size match, we can use the
-                                // same ABI as the scalar variant with the reserved niche.
-                                match st[i].abi() {
-                                    Abi::Scalar(_) => Abi::Scalar(niche_scalar),
-                                    Abi::ScalarPair(first, second) => {
-                                        // Only the niche is guaranteed to be initialised,
-                                        // so use union layout for the other primitive.
-                                        if offset.bytes() == 0 {
-                                            Abi::ScalarPair(niche_scalar, second.to_union())
-                                        } else {
-                                            Abi::ScalarPair(first.to_union(), niche_scalar)
+                                // Determine if it'll fit after the niche.
+                                let this_align = layout.align.abi;
+                                let this_offset = (niche_offset + niche_size).align_to(this_align);
+
+                                if this_offset + layout.size > size {
+                                    return false;
+                                }
+
+                                // It'll fit, but we need to make some adjustments.
+                                match layout.fields {
+                                    FieldsShape::Arbitrary { ref mut offsets, .. } => {
+                                        for (j, offset) in offsets.iter_mut().enumerate() {
+                                            if !variants[i][j].is_zst() {
+                                                *offset += this_offset;
+                                            }
                                         }
                                     }
-                                    _ => Abi::Aggregate { sized: true },
+                                    _ => {
+                                        panic!("Layout of fields should be Arbitrary for variants")
+                                    }
                                 }
-                            } else {
-                                Abi::Aggregate { sized: true }
-                            };
 
-                            let largest_niche = Niche::from_scalar(dl, offset, niche_scalar);
-
-                            niche_filling_layout = Some(LayoutS {
-                                variants: Variants::Multiple {
-                                    tag: niche_scalar,
-                                    tag_encoding: TagEncoding::Niche {
-                                        untagged_variant: i,
-                                        niche_variants,
-                                        niche_start,
-                                    },
-                                    tag_field: 0,
-                                    variants: st,
-                                },
-                                fields: FieldsShape::Arbitrary {
-                                    offsets: vec![offset],
-                                    memory_index: vec![0],
-                                },
-                                abi,
-                                largest_niche,
-                                size,
-                                align,
+                                // It can't be a Scalar or ScalarPair because the offset isn't 0.
+                                if !layout.abi.is_uninhabited() {
+                                    layout.abi = Abi::Aggregate { sized: true };
+                                }
+                                layout.size += this_offset;
+
+                                true
                             });
+
+                        if !all_variants_fit {
+                            return Ok(None);
                         }
-                    }
-                }
+
+                        let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
+
+                        let others_zst = variant_layouts.iter_enumerated().all(|(i, layout)| {
+                            i == largest_variant_index || layout.size == Size::ZERO
+                        });
+                        let same_size = size == variant_layouts[largest_variant_index].size;
+                        let same_align = align == variant_layouts[largest_variant_index].align;
+
+                        let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
+                            Abi::Uninhabited
+                        } else if same_size && same_align && others_zst {
+                            match variant_layouts[largest_variant_index].abi {
+                                // When the total alignment and size match, we can use the
+                                // same ABI as the scalar variant with the reserved niche.
+                                Abi::Scalar(_) => Abi::Scalar(niche_scalar),
+                                Abi::ScalarPair(first, second) => {
+                                    // Only the niche is guaranteed to be initialised,
+                                    // so use union layouts for the other primitive.
+                                    if niche_offset == Size::ZERO {
+                                        Abi::ScalarPair(niche_scalar, second.to_union())
+                                    } else {
+                                        Abi::ScalarPair(first.to_union(), niche_scalar)
+                                    }
+                                }
+                                _ => Abi::Aggregate { sized: true },
+                            }
+                        } else {
+                            Abi::Aggregate { sized: true }
+                        };
+
+                        let layout = LayoutS {
+                            variants: Variants::Multiple {
+                                tag: niche_scalar,
+                                tag_encoding: TagEncoding::Niche {
+                                    untagged_variant: largest_variant_index,
+                                    niche_variants,
+                                    niche_start,
+                                },
+                                tag_field: 0,
+                                variants: IndexVec::new(),
+                            },
+                            fields: FieldsShape::Arbitrary {
+                                offsets: vec![niche_offset],
+                                memory_index: vec![0],
+                            },
+                            abi,
+                            largest_niche,
+                            size,
+                            align,
+                        };
+
+                        Ok(Some(TmpLayout { layout, variants: variant_layouts }))
+                    };
+
+                let niche_filling_layout = calculate_niche_filling_layout()?;
 
                 let (mut min, mut max) = (i128::MAX, i128::MIN);
                 let discr_type = def.repr().discr_type();
@@ -1425,15 +1485,12 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
                 let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
 
-                let layout_variants =
-                    layout_variants.into_iter().map(|v| tcx.intern_layout(v)).collect();
-
                 let tagged_layout = LayoutS {
                     variants: Variants::Multiple {
                         tag,
                         tag_encoding: TagEncoding::Direct,
                         tag_field: 0,
-                        variants: layout_variants,
+                        variants: IndexVec::new(),
                     },
                     fields: FieldsShape::Arbitrary {
                         offsets: vec![Size::ZERO],
@@ -1445,20 +1502,45 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                     size,
                 };
 
-                let best_layout = match (tagged_layout, niche_filling_layout) {
-                    (tagged_layout, Some(niche_filling_layout)) => {
+                let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
+
+                let mut best_layout = match (tagged_layout, niche_filling_layout) {
+                    (tl, Some(nl)) => {
                         // Pick the smaller layout; otherwise,
                         // pick the layout with the larger niche; otherwise,
                         // pick tagged as it has simpler codegen.
-                        cmp::min_by_key(tagged_layout, niche_filling_layout, |layout| {
-                            let niche_size = layout.largest_niche.map_or(0, |n| n.available(dl));
-                            (layout.size, cmp::Reverse(niche_size))
-                        })
+                        use Ordering::*;
+                        let niche_size = |tmp_l: &TmpLayout<'_>| {
+                            tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
+                        };
+                        match (
+                            tl.layout.size.cmp(&nl.layout.size),
+                            niche_size(&tl).cmp(&niche_size(&nl)),
+                        ) {
+                            (Greater, _) => nl,
+                            (Equal, Less) => nl,
+                            _ => tl,
+                        }
                     }
-                    (tagged_layout, None) => tagged_layout,
+                    (tl, None) => tl,
+                };
+
+                // Now we can intern the variant layouts and store them in the enum layout.
+                best_layout.layout.variants = match best_layout.layout.variants {
+                    Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
+                        tag,
+                        tag_encoding,
+                        tag_field,
+                        variants: best_layout
+                            .variants
+                            .into_iter()
+                            .map(|layout| tcx.intern_layout(layout))
+                            .collect(),
+                    },
+                    _ => bug!(),
                 };
 
-                tcx.intern_layout(best_layout)
+                tcx.intern_layout(best_layout.layout)
             }
 
             // Types with no meaningful known layout.
diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index d6bb7c6c4fc..2077cf71b2e 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -2532,7 +2532,8 @@ mod size_asserts {
     // These are in alphabetical order, which is easy to maintain.
     static_assert_size!(Crate, 72); // frequently moved by-value
     static_assert_size!(DocFragment, 32);
-    static_assert_size!(GenericArg, 64);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(GenericArg, 56);
     static_assert_size!(GenericArgs, 32);
     static_assert_size!(GenericParamDef, 56);
     static_assert_size!(Item, 56);
diff --git a/src/test/ui/stats/hir-stats.rs b/src/test/ui/stats/hir-stats.rs
index a24b3ada57e..5102574d4be 100644
--- a/src/test/ui/stats/hir-stats.rs
+++ b/src/test/ui/stats/hir-stats.rs
@@ -1,6 +1,7 @@
 // check-pass
 // compile-flags: -Zhir-stats
 // only-x86_64
+// ignore-stage1
 
 // The aim here is to include at least one of every different type of top-level
 // AST/HIR node reported by `-Zhir-stats`.
diff --git a/src/test/ui/stats/hir-stats.stderr b/src/test/ui/stats/hir-stats.stderr
index 0a736f7be83..c8ceb6ff22d 100644
--- a/src/test/ui/stats/hir-stats.stderr
+++ b/src/test/ui/stats/hir-stats.stderr
@@ -21,39 +21,39 @@ ast-stats-1 - MacCall                   32 ( 0.4%)             1
 ast-stats-1 - Expr                      96 ( 1.1%)             3
 ast-stats-1 Param                    160 ( 1.9%)             4            40
 ast-stats-1 FnDecl                   200 ( 2.4%)             5            40
-ast-stats-1 Variant                  240 ( 2.8%)             2           120
+ast-stats-1 Variant                  240 ( 2.9%)             2           120
 ast-stats-1 Block                    288 ( 3.4%)             6            48
 ast-stats-1 GenericBound             352 ( 4.2%)             4            88
 ast-stats-1 - Trait                    352 ( 4.2%)             4
 ast-stats-1 AssocItem                416 ( 4.9%)             4           104
 ast-stats-1 - TyAlias                  208 ( 2.5%)             2
 ast-stats-1 - Fn                       208 ( 2.5%)             2
-ast-stats-1 GenericParam             520 ( 6.1%)             5           104
-ast-stats-1 PathSegment              720 ( 8.5%)            30            24
-ast-stats-1 Expr                     832 ( 9.8%)             8           104
+ast-stats-1 GenericParam             480 ( 5.7%)             5            96
+ast-stats-1 PathSegment              720 ( 8.6%)            30            24
+ast-stats-1 Expr                     832 ( 9.9%)             8           104
 ast-stats-1 - Path                     104 ( 1.2%)             1
 ast-stats-1 - Match                    104 ( 1.2%)             1
 ast-stats-1 - Struct                   104 ( 1.2%)             1
 ast-stats-1 - Lit                      208 ( 2.5%)             2
 ast-stats-1 - Block                    312 ( 3.7%)             3
-ast-stats-1 Pat                      840 ( 9.9%)             7           120
+ast-stats-1 Pat                      840 (10.0%)             7           120
 ast-stats-1 - Struct                   120 ( 1.4%)             1
 ast-stats-1 - Wild                     120 ( 1.4%)             1
 ast-stats-1 - Ident                    600 ( 7.1%)             5
-ast-stats-1 Ty                     1_344 (15.9%)            14            96
+ast-stats-1 Ty                     1_344 (16.0%)            14            96
 ast-stats-1 - Rptr                      96 ( 1.1%)             1
 ast-stats-1 - Ptr                       96 ( 1.1%)             1
 ast-stats-1 - ImplicitSelf             192 ( 2.3%)             2
 ast-stats-1 - Path                     960 (11.4%)            10
-ast-stats-1 Item                   1_656 (19.6%)             9           184
+ast-stats-1 Item                   1_656 (19.7%)             9           184
 ast-stats-1 - Trait                    184 ( 2.2%)             1
 ast-stats-1 - Enum                     184 ( 2.2%)             1
 ast-stats-1 - ForeignMod               184 ( 2.2%)             1
 ast-stats-1 - Impl                     184 ( 2.2%)             1
 ast-stats-1 - Fn                       368 ( 4.4%)             2
-ast-stats-1 - Use                      552 ( 6.5%)             3
+ast-stats-1 - Use                      552 ( 6.6%)             3
 ast-stats-1 ----------------------------------------------------------------
-ast-stats-1 Total                  8_456
+ast-stats-1 Total                  8_416
 ast-stats-1
 ast-stats-2 POST EXPANSION AST STATS
 ast-stats-2 Name                Accumulated Size         Count     Item Size
@@ -86,12 +86,12 @@ ast-stats-2 - Trait                    352 ( 3.8%)             4
 ast-stats-2 AssocItem                416 ( 4.5%)             4           104
 ast-stats-2 - TyAlias                  208 ( 2.3%)             2
 ast-stats-2 - Fn                       208 ( 2.3%)             2
-ast-stats-2 GenericParam             520 ( 5.7%)             5           104
-ast-stats-2 PathSegment              792 ( 8.6%)            33            24
-ast-stats-2 Pat                      840 ( 9.1%)             7           120
+ast-stats-2 GenericParam             480 ( 5.2%)             5            96
+ast-stats-2 PathSegment              792 ( 8.7%)            33            24
+ast-stats-2 Pat                      840 ( 9.2%)             7           120
 ast-stats-2 - Struct                   120 ( 1.3%)             1
 ast-stats-2 - Wild                     120 ( 1.3%)             1
-ast-stats-2 - Ident                    600 ( 6.5%)             5
+ast-stats-2 - Ident                    600 ( 6.6%)             5
 ast-stats-2 Expr                     936 (10.2%)             9           104
 ast-stats-2 - Path                     104 ( 1.1%)             1
 ast-stats-2 - Match                    104 ( 1.1%)             1
@@ -99,12 +99,12 @@ ast-stats-2 - Struct                   104 ( 1.1%)             1
 ast-stats-2 - InlineAsm                104 ( 1.1%)             1
 ast-stats-2 - Lit                      208 ( 2.3%)             2
 ast-stats-2 - Block                    312 ( 3.4%)             3
-ast-stats-2 Ty                     1_344 (14.6%)            14            96
+ast-stats-2 Ty                     1_344 (14.7%)            14            96
 ast-stats-2 - Rptr                      96 ( 1.0%)             1
 ast-stats-2 - Ptr                       96 ( 1.0%)             1
 ast-stats-2 - ImplicitSelf             192 ( 2.1%)             2
 ast-stats-2 - Path                     960 (10.5%)            10
-ast-stats-2 Item                   2_024 (22.0%)            11           184
+ast-stats-2 Item                   2_024 (22.1%)            11           184
 ast-stats-2 - Trait                    184 ( 2.0%)             1
 ast-stats-2 - Enum                     184 ( 2.0%)             1
 ast-stats-2 - ExternCrate              184 ( 2.0%)             1
@@ -113,7 +113,7 @@ ast-stats-2 - Impl                     184 ( 2.0%)             1
 ast-stats-2 - Fn                       368 ( 4.0%)             2
 ast-stats-2 - Use                      736 ( 8.0%)             4
 ast-stats-2 ----------------------------------------------------------------
-ast-stats-2 Total                  9_184
+ast-stats-2 Total                  9_144
 ast-stats-2
 hir-stats HIR STATS
 hir-stats Name                Accumulated Size         Count     Item Size
@@ -121,7 +121,7 @@ hir-stats ----------------------------------------------------------------
 hir-stats ForeignItemRef            24 ( 0.2%)             1            24
 hir-stats Mod                       32 ( 0.3%)             1            32
 hir-stats ExprField                 40 ( 0.4%)             1            40
-hir-stats TraitItemRef              56 ( 0.5%)             2            28
+hir-stats TraitItemRef              56 ( 0.6%)             2            28
 hir-stats Param                     64 ( 0.6%)             2            32
 hir-stats Local                     64 ( 0.6%)             1            64
 hir-stats InlineAsm                 72 ( 0.7%)             1            72
@@ -135,11 +135,11 @@ hir-stats - Semi                      32 ( 0.3%)             1
 hir-stats - Expr                      32 ( 0.3%)             1
 hir-stats FnDecl                   120 ( 1.2%)             3            40
 hir-stats Attribute                128 ( 1.3%)             4            32
+hir-stats GenericArg               128 ( 1.3%)             4            32
+hir-stats - Type                      32 ( 0.3%)             1
+hir-stats - Lifetime                  96 ( 0.9%)             3
 hir-stats GenericArgs              144 ( 1.4%)             3            48
 hir-stats Variant                  160 ( 1.6%)             2            80
-hir-stats GenericArg               160 ( 1.6%)             4            40
-hir-stats - Type                      40 ( 0.4%)             1
-hir-stats - Lifetime                 120 ( 1.2%)             3
 hir-stats GenericBound             192 ( 1.9%)             4            48
 hir-stats - Trait                    192 ( 1.9%)             4
 hir-stats WherePredicate           216 ( 2.1%)             3            72
@@ -151,7 +151,7 @@ hir-stats - Wild                      88 ( 0.9%)             1
 hir-stats - Struct                    88 ( 0.9%)             1
 hir-stats - Binding                  264 ( 2.6%)             3
 hir-stats Generics                 560 ( 5.5%)            10            56
-hir-stats Expr                     768 ( 7.5%)            12            64
+hir-stats Expr                     768 ( 7.6%)            12            64
 hir-stats - Path                      64 ( 0.6%)             1
 hir-stats - Struct                    64 ( 0.6%)             1
 hir-stats - Match                     64 ( 0.6%)             1
@@ -173,5 +173,5 @@ hir-stats - Path                     936 ( 9.2%)            13
 hir-stats Path                   1_536 (15.1%)            32            48
 hir-stats PathSegment            2_240 (22.0%)            40            56
 hir-stats ----------------------------------------------------------------
-hir-stats Total                 10_200
+hir-stats Total                 10_168
 hir-stats
diff --git a/src/test/ui/structs-enums/type-sizes.rs b/src/test/ui/structs-enums/type-sizes.rs
index 73a11a5e743..7a23f13630a 100644
--- a/src/test/ui/structs-enums/type-sizes.rs
+++ b/src/test/ui/structs-enums/type-sizes.rs
@@ -120,6 +120,54 @@ pub enum AlwaysTaggedBecauseItHasNoNiche {
     B
 }
 
+pub enum NicheFilledMultipleFields {
+    A(bool, u8),
+    B(u8),
+    C(u8),
+    D(bool),
+    E,
+    F,
+    G,
+}
+
+struct BoolInTheMiddle(std::num::NonZeroU16, bool, u8);
+
+enum NicheWithData {
+    A,
+    B([u16; 5]),
+    Largest { a1: u32, a2: BoolInTheMiddle, a3: u32 },
+    C,
+    D(u32, u32),
+}
+
+// A type with almost 2^16 invalid values.
+#[repr(u16)]
+pub enum NicheU16 {
+    _0,
+}
+
+pub enum EnumManyVariant<X> {
+    Dataful(u8, X),
+
+    // 0x100 niche variants.
+    _00, _01, _02, _03, _04, _05, _06, _07, _08, _09, _0A, _0B, _0C, _0D, _0E, _0F,
+    _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _1A, _1B, _1C, _1D, _1E, _1F,
+    _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _2A, _2B, _2C, _2D, _2E, _2F,
+    _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _3A, _3B, _3C, _3D, _3E, _3F,
+    _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _4A, _4B, _4C, _4D, _4E, _4F,
+    _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _5A, _5B, _5C, _5D, _5E, _5F,
+    _60, _61, _62, _63, _64, _65, _66, _67, _68, _69, _6A, _6B, _6C, _6D, _6E, _6F,
+    _70, _71, _72, _73, _74, _75, _76, _77, _78, _79, _7A, _7B, _7C, _7D, _7E, _7F,
+    _80, _81, _82, _83, _84, _85, _86, _87, _88, _89, _8A, _8B, _8C, _8D, _8E, _8F,
+    _90, _91, _92, _93, _94, _95, _96, _97, _98, _99, _9A, _9B, _9C, _9D, _9E, _9F,
+    _A0, _A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8, _A9, _AA, _AB, _AC, _AD, _AE, _AF,
+    _B0, _B1, _B2, _B3, _B4, _B5, _B6, _B7, _B8, _B9, _BA, _BB, _BC, _BD, _BE, _BF,
+    _C0, _C1, _C2, _C3, _C4, _C5, _C6, _C7, _C8, _C9, _CA, _CB, _CC, _CD, _CE, _CF,
+    _D0, _D1, _D2, _D3, _D4, _D5, _D6, _D7, _D8, _D9, _DA, _DB, _DC, _DD, _DE, _DF,
+    _E0, _E1, _E2, _E3, _E4, _E5, _E6, _E7, _E8, _E9, _EA, _EB, _EC, _ED, _EE, _EF,
+    _F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF,
+}
+
 pub fn main() {
     assert_eq!(size_of::<u8>(), 1 as usize);
     assert_eq!(size_of::<u32>(), 4 as usize);
@@ -170,4 +218,35 @@ pub fn main() {
     assert_eq!(size_of::<AlwaysTaggedBecauseItHasNoNiche>(), 8);
     assert_eq!(size_of::<Option<AlwaysTaggedBecauseItHasNoNiche>>(), 8);
     assert_eq!(size_of::<Option<Option<AlwaysTaggedBecauseItHasNoNiche>>>(), 8);
+
+    assert_eq!(size_of::<NicheFilledMultipleFields>(), 2);
+    assert_eq!(size_of::<Option<NicheFilledMultipleFields>>(), 2);
+    assert_eq!(size_of::<Option<Option<NicheFilledMultipleFields>>>(), 2);
+
+    struct S1{ a: u16, b: std::num::NonZeroU16, c: u16, d: u8, e: u32, f: u64, g:[u8;2] }
+    assert_eq!(size_of::<S1>(), 24);
+    assert_eq!(size_of::<Option<S1>>(), 24);
+
+    assert_eq!(size_of::<NicheWithData>(), 12);
+    assert_eq!(size_of::<Option<NicheWithData>>(), 12);
+    assert_eq!(size_of::<Option<Option<NicheWithData>>>(), 12);
+    assert_eq!(
+        size_of::<Option<Option2<&(), Option<NicheWithData>>>>(),
+        size_of::<(&(), NicheWithData)>()
+    );
+
+    pub enum FillPadding { A(std::num::NonZeroU8, u32), B }
+    assert_eq!(size_of::<FillPadding>(), 8);
+    assert_eq!(size_of::<Option<FillPadding>>(), 8);
+    assert_eq!(size_of::<Option<Option<FillPadding>>>(), 8);
+
+    assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8), u16>>(), 4);
+    assert_eq!(size_of::<Option<Result<(std::num::NonZeroU8, u8, u8), u16>>>(), 4);
+    assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8, u8), u16>>(), 4);
+
+    assert_eq!(size_of::<EnumManyVariant<u16>>(), 6);
+    assert_eq!(size_of::<EnumManyVariant<NicheU16>>(), 4);
+    assert_eq!(size_of::<EnumManyVariant<Option<NicheU16>>>(), 4);
+    assert_eq!(size_of::<EnumManyVariant<Option2<NicheU16,u8>>>(), 6);
+    assert_eq!(size_of::<EnumManyVariant<Option<(NicheU16,u8)>>>(), 6);
 }