3 files changed, 55 insertions, 12 deletions
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index bbb89576ef7..5b58d7f43b2 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -678,15 +678,19 @@ pub enum Constructor<Cx: PatCx> {
     Or,
     /// Wildcard pattern.
     Wildcard,
+    /// Never pattern. Only used in `WitnessPat`. An actual never pattern should be lowered as
+    /// `Wildcard`.
+    Never,
     /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
-    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
+    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`. Only
+    /// used in `WitnessPat`.
     NonExhaustive,
-    /// Fake extra constructor for variants that should not be mentioned in diagnostics.
-    /// We use this for variants behind an unstable gate as well as
-    /// `#[doc(hidden)]` ones.
+    /// Fake extra constructor for variants that should not be mentioned in diagnostics. We use this
+    /// for variants behind an unstable gate as well as `#[doc(hidden)]` ones. Only used in
+    /// `WitnessPat`.
     Hidden,
     /// Fake extra constructor for constructors that are not seen in the matrix, as explained at the
-    /// top of the file.
+    /// top of the file. Only used for specialization.
     Missing,
     /// Fake extra constructor that indicates and empty field that is private. When we encounter one
     /// we skip the column entirely so we don't observe its emptiness. Only used for specialization.
@@ -708,6 +712,7 @@ impl<Cx: PatCx> Clone for Constructor<Cx> {
             Constructor::Str(value) => Constructor::Str(value.clone()),
             Constructor::Opaque(inner) => Constructor::Opaque(inner.clone()),
             Constructor::Or => Constructor::Or,
+            Constructor::Never => Constructor::Never,
             Constructor::Wildcard => Constructor::Wildcard,
             Constructor::NonExhaustive => Constructor::NonExhaustive,
             Constructor::Hidden => Constructor::Hidden,
@@ -1040,10 +1045,32 @@ impl<Cx: PatCx> ConstructorSet<Cx> {
                 // In a `MaybeInvalid` place even an empty pattern may be reachable. We therefore
                 // add a dummy empty constructor here, which will be ignored if the place is
                 // `ValidOnly`.
-                missing_empty.push(NonExhaustive);
+                missing_empty.push(Never);
             }
         }
 
         SplitConstructorSet { present, missing, missing_empty }
     }
+
+    /// Whether this set only contains empty constructors.
+    pub(crate) fn all_empty(&self) -> bool {
+        match self {
+            ConstructorSet::Bool
+            | ConstructorSet::Integers { .. }
+            | ConstructorSet::Ref
+            | ConstructorSet::Union
+            | ConstructorSet::Unlistable => false,
+            ConstructorSet::NoConstructors => true,
+            ConstructorSet::Struct { empty } => *empty,
+            ConstructorSet::Variants { variants, non_exhaustive } => {
+                !*non_exhaustive
+                    && variants
+                        .iter()
+                        .all(|visibility| matches!(visibility, VariantVisibility::Empty))
+            }
+            ConstructorSet::Slice { array_len, subtype_is_empty } => {
+                *subtype_is_empty && matches!(array_len, Some(1..))
+            }
+        }
+    }
 }
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index e3667d44bc9..e7eed673c94 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -208,6 +208,7 @@ impl<Cx: PatCx> fmt::Debug for DeconstructedPat<Cx> {
                 }
                 Ok(())
             }
+            Never => write!(f, "!"),
             Wildcard | Missing | NonExhaustive | Hidden | PrivateUninhabited => {
                 write!(f, "_ : {:?}", pat.ty())
             }
@@ -311,18 +312,24 @@ impl<Cx: PatCx> WitnessPat<Cx> {
     pub(crate) fn new(ctor: Constructor<Cx>, fields: Vec<Self>, ty: Cx::Ty) -> Self {
         Self { ctor, fields, ty }
     }
-    pub(crate) fn wildcard(ty: Cx::Ty) -> Self {
-        Self::new(Wildcard, Vec::new(), ty)
+    /// Create a wildcard pattern for this type. If the type is empty, we create a `!` pattern.
+    pub(crate) fn wildcard(cx: &Cx, ty: Cx::Ty) -> Self {
+        let is_empty = cx.ctors_for_ty(&ty).is_ok_and(|ctors| ctors.all_empty());
+        let ctor = if is_empty { Never } else { Wildcard };
+        Self::new(ctor, Vec::new(), ty)
     }
 
     /// Construct a pattern that matches everything that starts with this constructor.
     /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
     /// `Some(_)`.
     pub(crate) fn wild_from_ctor(cx: &Cx, ctor: Constructor<Cx>, ty: Cx::Ty) -> Self {
+        if matches!(ctor, Wildcard) {
+            return Self::wildcard(cx, ty);
+        }
         let fields = cx
             .ctor_sub_tys(&ctor, &ty)
             .filter(|(_, PrivateUninhabitedField(skip))| !skip)
-            .map(|(ty, _)| Self::wildcard(ty))
+            .map(|(ty, _)| Self::wildcard(cx, ty))
             .collect();
         Self::new(ctor, fields, ty)
     }
@@ -334,6 +341,14 @@ impl<Cx: PatCx> WitnessPat<Cx> {
         &self.ty
     }
 
+    pub fn is_never_pattern(&self) -> bool {
+        match self.ctor() {
+            Never => true,
+            Or => self.fields.iter().all(|p| p.is_never_pattern()),
+            _ => self.fields.iter().any(|p| p.is_never_pattern()),
+        }
+    }
+
     pub fn iter_fields(&self) -> impl Iterator<Item = &WitnessPat<Cx>> {
         self.fields.iter()
     }
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
index fd51fbedeef..eedc00a5613 100644
--- a/compiler/rustc_pattern_analysis/src/rustc.rs
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -247,7 +247,7 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
                 _ => bug!("bad slice pattern {:?} {:?}", ctor, ty),
             },
             Bool(..) | IntRange(..) | F32Range(..) | F64Range(..) | Str(..) | Opaque(..)
-            | NonExhaustive | Hidden | Missing | PrivateUninhabited | Wildcard => &[],
+            | Never | NonExhaustive | Hidden | Missing | PrivateUninhabited | Wildcard => &[],
             Or => {
                 bug!("called `Fields::wildcards` on an `Or` ctor")
             }
@@ -275,7 +275,7 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
             Ref => 1,
             Slice(slice) => slice.arity(),
             Bool(..) | IntRange(..) | F32Range(..) | F64Range(..) | Str(..) | Opaque(..)
-            | NonExhaustive | Hidden | Missing | PrivateUninhabited | Wildcard => 0,
+            | Never | NonExhaustive | Hidden | Missing | PrivateUninhabited | Wildcard => 0,
             Or => bug!("The `Or` constructor doesn't have a fixed arity"),
         }
     }
@@ -824,7 +824,8 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
                 }
             }
             &Str(value) => PatKind::Constant { value },
-            Wildcard | NonExhaustive | Hidden | PrivateUninhabited => PatKind::Wild,
+            Never if self.tcx.features().never_patterns => PatKind::Never,
+            Never | Wildcard | NonExhaustive | Hidden | PrivateUninhabited => PatKind::Wild,
             Missing { .. } => bug!(
                 "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
                 `Missing` should have been processed in `apply_constructors`"